diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/types.go b/staging/src/k8s.io/apiserver/pkg/apis/config/types.go index a719fecf292a..bae49e8eee29 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/types.go @@ -26,11 +26,23 @@ import ( /* EncryptionConfiguration stores the complete configuration for encryption providers. -example: +It also allows the use of wildcards to specify the resources that should be encrypted. +Use '*.' to encrypt all resources within a group or '*.*' to encrypt all resources. +'*.' can be used to encrypt all resource in the core group. '*.*' will encrypt all +resources, even custom resources that are added after API server start. +Use of wildcards that overlap within the same resource list or across multiple +entries are not allowed since part of the configuration would be ineffective. +Resource lists are processed in order, with earlier lists taking precedence. + +Example: kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 resources: + - resources: + - events + providers: + - identity: {} # do not encrypt events even though *.* is specified below - resources: - secrets - configmaps @@ -40,6 +52,20 @@ example: keys: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== + - resources: + - '*.apps' + providers: + - aescbc: + keys: + - name: key2 + secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg== + - resources: + - '*.*' + providers: + - aescbc: + keys: + - name: key3 + secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw== */ type EncryptionConfiguration struct { metav1.TypeMeta @@ -50,10 +76,13 @@ type EncryptionConfiguration struct { // ResourceConfiguration stores per resource configuration. type ResourceConfiguration struct { // resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource. - // eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas) + // eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas. + // Use '*.*' to encrypt all resources and '*.' to encrypt all resources in a specific group. + // eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'. + // eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc). Resources []string // providers is a list of transformers to be used for reading and writing the resources to disk. - // eg: aesgcm, aescbc, secretbox, identity. + // eg: aesgcm, aescbc, secretbox, identity, kms. Providers []ProviderConfiguration } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go index fe86305350d0..7aced8cf6288 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/v1/types.go @@ -26,11 +26,23 @@ import ( /* EncryptionConfiguration stores the complete configuration for encryption providers. -example: +It also allows the use of wildcards to specify the resources that should be encrypted. +Use '*.' to encrypt all resources within a group or '*.*' to encrypt all resources. +'*.' can be used to encrypt all resource in the core group. '*.*' will encrypt all +resources, even custom resources that are added after API server start. +Use of wildcards that overlap within the same resource list or across multiple +entries are not allowed since part of the configuration would be ineffective. +Resource lists are processed in order, with earlier lists taking precedence. + +Example: kind: EncryptionConfiguration apiVersion: apiserver.config.k8s.io/v1 resources: + - resources: + - events + providers: + - identity: {} # do not encrypt events even though *.* is specified below - resources: - secrets - configmaps @@ -40,6 +52,20 @@ example: keys: - name: key1 secret: c2VjcmV0IGlzIHNlY3VyZQ== + - resources: + - '*.apps' + providers: + - aescbc: + keys: + - name: key2 + secret: c2VjcmV0IGlzIHNlY3VyZSwgb3IgaXMgaXQ/Cg== + - resources: + - '*.*' + providers: + - aescbc: + keys: + - name: key3 + secret: c2VjcmV0IGlzIHNlY3VyZSwgSSB0aGluaw== */ type EncryptionConfiguration struct { metav1.TypeMeta @@ -50,10 +76,13 @@ type EncryptionConfiguration struct { // ResourceConfiguration stores per resource configuration. type ResourceConfiguration struct { // resources is a list of kubernetes resources which have to be encrypted. The resource names are derived from `resource` or `resource.group` of the group/version/resource. - // eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas) + // eg: pandas.awesome.bears.example is a custom resource with 'group': awesome.bears.example, 'resource': pandas. + // Use '*.*' to encrypt all resources and '*.' to encrypt all resources in a specific group. + // eg: '*.awesome.bears.example' will encrypt all resources in the group 'awesome.bears.example'. + // eg: '*.' will encrypt all resources in the core group (such as pods, configmaps, etc). Resources []string `json:"resources"` // providers is a list of transformers to be used for reading and writing the resources to disk. - // eg: aesgcm, aescbc, secretbox, identity. + // eg: aesgcm, aescbc, secretbox, identity, kms. Providers []ProviderConfiguration `json:"providers"` } diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation.go b/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation.go index 7da5291e79d4..90708472a830 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation.go @@ -23,6 +23,7 @@ import ( "net/url" "strings" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/apiserver/pkg/apis/config" @@ -34,7 +35,7 @@ const ( unsupportedSchemeErrFmt = "unsupported scheme %q for KMS provider, only unix is supported" unsupportedKMSAPIVersionErrFmt = "unsupported apiVersion %s for KMS provider, only v1 and v2 are supported" atLeastOneRequiredErrFmt = "at least one %s is required" - invalidURLErrFmt = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL" + invalidURLErrFmt = "invalid endpoint for kms provider, error: %v" mandatoryFieldErrFmt = "%s is a mandatory field for a %s" base64EncodingErr = "secrets must be base64 encoded" zeroOrNegativeErrFmt = "%s should be a positive value" @@ -42,6 +43,14 @@ const ( encryptionConfigNilErr = "EncryptionConfiguration can't be nil" invalidKMSConfigNameErrFmt = "invalid KMS provider name %s, must not contain ':'" duplicateKMSConfigNameErrFmt = "duplicate KMS provider name %s, names must be unique" + eventsGroupErr = "'*.events.k8s.io' objects are stored using the 'events' API group in etcd. Use 'events' instead in the config file" + extensionsGroupErr = "'extensions' group has been removed and cannot be used for encryption" + starResourceErr = "use '*.' to encrypt all the resources from core API group or *.* to encrypt all resources" + overlapErr = "using overlapping resources such as 'secrets' and '*.' in the same resource list is not allowed as they will be masked" + nonRESTAPIResourceErr = "resources which do not have REST API/s cannot be encrypted" + resourceNameErr = "resource name should not contain capital letters" + resourceAcrossGroupErr = "encrypting the same resource across groups is not supported" + duplicateResourceErr = "the same resource cannot be specified multiple times" ) var ( @@ -59,7 +68,7 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b allErrs := field.ErrorList{} if c == nil { - allErrs = append(allErrs, field.Required(root, "EncryptionConfiguration can't be nil")) + allErrs = append(allErrs, field.Required(root, encryptionConfigNilErr)) return allErrs } @@ -78,6 +87,9 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b allErrs = append(allErrs, field.Required(r, fmt.Sprintf(atLeastOneRequiredErrFmt, r))) } + allErrs = append(allErrs, validateResourceOverlap(conf.Resources, r)...) + allErrs = append(allErrs, validateResourceNames(conf.Resources, r)...) + if len(conf.Providers) == 0 { allErrs = append(allErrs, field.Required(p, fmt.Sprintf(atLeastOneRequiredErrFmt, p))) } @@ -103,6 +115,175 @@ func ValidateEncryptionConfiguration(c *config.EncryptionConfiguration, reload b return allErrs } +var anyGroupAnyResource = schema.GroupResource{ + Group: "*", + Resource: "*", +} + +func validateResourceOverlap(resources []string, fieldPath *field.Path) field.ErrorList { + if len(resources) < 2 { // cannot have overlap with a single resource + return nil + } + + var allErrs field.ErrorList + + r := make([]schema.GroupResource, 0, len(resources)) + for _, resource := range resources { + r = append(r, schema.ParseGroupResource(resource)) + } + + var hasOverlap, hasDuplicate bool + + for i, r1 := range r { + for j, r2 := range r { + if i == j { + continue + } + + if r1 == r2 && !hasDuplicate { + hasDuplicate = true + continue + } + + if hasOverlap { + continue + } + + if r1 == anyGroupAnyResource { + hasOverlap = true + continue + } + + if r1.Group != r2.Group { + continue + } + + if r1.Resource == "*" || r2.Resource == "*" { + hasOverlap = true + continue + } + } + } + + if hasDuplicate { + allErrs = append( + allErrs, + field.Invalid( + fieldPath, + resources, + duplicateResourceErr, + ), + ) + } + + if hasOverlap { + allErrs = append( + allErrs, + field.Invalid( + fieldPath, + resources, + overlapErr, + ), + ) + } + + return allErrs +} + +func validateResourceNames(resources []string, fieldPath *field.Path) field.ErrorList { + var allErrs field.ErrorList + + for j, res := range resources { + jj := fieldPath.Index(j) + + // check if resource name has capital letters + if hasCapital(res) { + allErrs = append( + allErrs, + field.Invalid( + jj, + resources[j], + resourceNameErr, + ), + ) + continue + } + + // check if resource is '*' + if res == "*" { + allErrs = append( + allErrs, + field.Invalid( + jj, + resources[j], + starResourceErr, + ), + ) + continue + } + + // check if resource is: + // 'apiserveripinfo' OR + // 'serviceipallocations' OR + // 'servicenodeportallocations' OR + if res == "apiserveripinfo" || + res == "serviceipallocations" || + res == "servicenodeportallocations" { + allErrs = append( + allErrs, + field.Invalid( + jj, + resources[j], + nonRESTAPIResourceErr, + ), + ) + continue + } + + // check if group is 'events.k8s.io' + gr := schema.ParseGroupResource(res) + if gr.Group == "events.k8s.io" { + allErrs = append( + allErrs, + field.Invalid( + jj, + resources[j], + eventsGroupErr, + ), + ) + continue + } + + // check if group is 'extensions' + if gr.Group == "extensions" { + allErrs = append( + allErrs, + field.Invalid( + jj, + resources[j], + extensionsGroupErr, + ), + ) + continue + } + + // disallow resource.* as encrypting the same resource across groups does not make sense + if gr.Group == "*" && gr.Resource != "*" { + allErrs = append( + allErrs, + field.Invalid( + jj, + resources[j], + resourceAcrossGroupErr, + ), + ) + continue + } + } + + return allErrs +} + func validateSingleProvider(provider config.ProviderConfiguration, fieldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} found := 0 @@ -225,7 +406,7 @@ func validateKMSEndpoint(c *config.KMSConfiguration, fieldPath *field.Path) fiel u, err := url.Parse(c.Endpoint) if err != nil { - return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf("invalid endpoint for kms provider, error: %v", err))) + return append(allErrs, field.Invalid(fieldPath, c.Endpoint, fmt.Sprintf(invalidURLErrFmt, err))) } if u.Scheme != "unix" { @@ -265,3 +446,7 @@ func validateKMSConfigName(c *config.KMSConfiguration, fieldPath *field.Path, km return allErrs } + +func hasCapital(input string) bool { + return strings.ToLower(input) != input +} diff --git a/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation_test.go b/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation_test.go index eeecf408c523..d57a3f5294bb 100644 --- a/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation_test.go +++ b/staging/src/k8s.io/apiserver/pkg/apis/config/validation/validation_test.go @@ -22,6 +22,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" @@ -370,6 +371,737 @@ func TestStructure(t *testing.T) { "foo", fmt.Sprintf(duplicateKMSConfigNameErrFmt, "foo")), }, }, + { + desc: "config should error when events.k8s.io group is used", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "events.events.k8s.io", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "events.events.k8s.io", + eventsGroupErr, + ), + }, + }, { + desc: "config should error when events.k8s.io group is used later in the list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "secrets", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + { + Resources: []string{ + "secret", + "events.events.k8s.io", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(1).Child("resources").Index(1), + "events.events.k8s.io", + eventsGroupErr, + ), + }, + }, + { + desc: "config should error when *.events.k8s.io group is used", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*.events.k8s.io", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "*.events.k8s.io", + eventsGroupErr, + ), + }, + }, + { + desc: "config should error when extensions group is used", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*.extensions", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "*.extensions", + extensionsGroupErr, + ), + }, + }, + { + desc: "config should error when foo.extensions group is used", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "foo.extensions", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "foo.extensions", + extensionsGroupErr, + ), + }, + }, + { + desc: "config should error when '*' resource is used", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "*", + starResourceErr, + ), + }, + }, + { + desc: "should error when resource name has capital letters", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "apiServerIPInfo", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "apiServerIPInfo", + resourceNameErr, + ), + }, + }, + { + desc: "should error when resource name is apiserveripinfo", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "apiserveripinfo", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "apiserveripinfo", + nonRESTAPIResourceErr, + ), + }, + }, + { + desc: "should error when resource name is serviceipallocations", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "serviceipallocations", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "serviceipallocations", + nonRESTAPIResourceErr, + ), + }, + }, + { + desc: "should error when resource name is servicenodeportallocations", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "servicenodeportallocations", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(0), + "servicenodeportallocations", + nonRESTAPIResourceErr, + ), + }, + }, + { + desc: "should not error when '*.apps' and '*.' are used within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*.apps", + "*.", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{}, + }, + { + desc: "should error when the same resource across groups is encrypted", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*.", + "foos.*", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources").Index(1), + "foos.*", + resourceAcrossGroupErr, + ), + }, + }, + { + desc: "should error when secrets are specified twice within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "secrets", + "secrets", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "secrets", + "secrets", + }, + duplicateResourceErr, + ), + }, + }, + { + desc: "should error once when secrets are specified many times within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "secrets", + "secrets", + "secrets", + "secrets", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "secrets", + "secrets", + "secrets", + "secrets", + }, + duplicateResourceErr, + ), + }, + }, + { + desc: "should error when secrets are specified twice within the same resource list, via dot", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "secrets", + "secrets.", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "secrets", + "secrets.", + }, + duplicateResourceErr, + ), + }, + }, + { + desc: "should error when '*.apps' and '*.' and '*.*' are used within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*.apps", + "*.", + "*.*", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "*.apps", + "*.", + "*.*", + }, + overlapErr, + ), + }, + }, + { + desc: "should not error when deployments.apps are specified with '*.' within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "deployments.apps", + "*.", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{}, + }, + { + desc: "should error when deployments.apps are specified with '*.apps' within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "deployments.apps", + "*.apps", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "deployments.apps", + "*.apps", + }, + overlapErr, + ), + }, + }, + { + desc: "should error when secrets are specified with '*.' within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "secrets", + "*.", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "secrets", + "*.", + }, + overlapErr, + ), + }, + }, + { + desc: "should error when pods are specified with '*.' within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "pods", + "*.", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "pods", + "*.", + }, + overlapErr, + ), + }, + }, + { + desc: "should error when other resources are specified with '*.*' within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "secrets", + "*.*", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "secrets", + "*.*", + }, + overlapErr, + ), + }, + }, + { + desc: "should error when both '*.' and '*.*' are used within the same resource list", + in: &config.EncryptionConfiguration{ + Resources: []config.ResourceConfiguration{ + { + Resources: []string{ + "*.", + "*.*", + }, + Providers: []config.ProviderConfiguration{ + { + KMS: &config.KMSConfiguration{ + Name: "foo", + Endpoint: "unix:///tmp/kms-provider.socket", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + CacheSize: &cacheSize, + APIVersion: "v1", + }, + }, + }, + }, + }, + }, + reload: false, + want: field.ErrorList{ + field.Invalid( + root.Index(0).Child("resources"), + []string{ + "*.", + "*.*", + }, + overlapErr, + ), + }, + }, } for _, tt := range testCases { @@ -505,7 +1237,7 @@ func TestKMSEndpoint(t *testing.T) { desc: "invalid url", in: &config.KMSConfiguration{Endpoint: "unix:///foo\n.socket"}, want: field.ErrorList{ - field.Invalid(endpointField, "unix:///foo\n.socket", fmt.Sprintf(invalidURLErrFmt, `"unix:///foo\n.socket"`)), + field.Invalid(endpointField, "unix:///foo\n.socket", fmt.Sprintf(invalidURLErrFmt, `parse "unix:///foo\n.socket": net/url: invalid control character in URL`)), }, }, } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go index 3f73a3d54262..e920411a9a13 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go @@ -218,8 +218,22 @@ func getTransformerOverridesAndKMSPluginProbes(ctx context.Context, config *apis for _, resource := range resourceConfig.Resources { resource := resource gr := schema.ParseGroupResource(resource) - resourceToPrefixTransformer[gr] = append( - resourceToPrefixTransformer[gr], transformers...) + + // check if resource is masked by *.group rule + anyResourceInGroup := schema.GroupResource{Group: gr.Group, Resource: "*"} + if _, masked := resourceToPrefixTransformer[anyResourceInGroup]; masked { + // an earlier rule already configured a transformer for *.group, masking this rule + // return error since this is not allowed + return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyResourceInGroup)) + } + + if _, masked := resourceToPrefixTransformer[anyGroupAnyResource]; masked { + // an earlier rule already configured a transformer for *.*, masking this rule + // return error since this is not allowed + return nil, nil, nil, fmt.Errorf("resource %q is masked by earlier rule %q", grYAMLString(gr), grYAMLString(anyGroupAnyResource)) + } + + resourceToPrefixTransformer[gr] = append(resourceToPrefixTransformer[gr], transformers...) } probes = append(probes, p...) @@ -777,10 +791,34 @@ func (s StaticTransformers) TransformerForResource(resource schema.GroupResource return transformerFromOverrides(s, resource) } +var anyGroupAnyResource = schema.GroupResource{ + Group: "*", + Resource: "*", +} + func transformerFromOverrides(transformerOverrides map[schema.GroupResource]value.Transformer, resource schema.GroupResource) value.Transformer { - transformer := transformerOverrides[resource] - if transformer == nil { - return identity.NewEncryptCheckTransformer() + if transformer := transformerOverrides[resource]; transformer != nil { + return transformer + } + + if transformer := transformerOverrides[schema.GroupResource{ + Group: resource.Group, + Resource: "*", + }]; transformer != nil { + return transformer + } + + if transformer := transformerOverrides[anyGroupAnyResource]; transformer != nil { + return transformer + } + + return identity.NewEncryptCheckTransformer() +} + +func grYAMLString(gr schema.GroupResource) string { + if gr.Group == "" && gr.Resource == "*" { + return "*." } - return transformer + + return gr.String() } diff --git a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go index d0f5ab38102f..ca573f8efefb 100644 --- a/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go +++ b/staging/src/k8s.io/apiserver/pkg/server/options/encryptionconfig/config_test.go @@ -21,6 +21,7 @@ import ( "context" "encoding/base64" "errors" + "reflect" "strings" "sync" "testing" @@ -41,6 +42,7 @@ import ( "k8s.io/component-base/metrics/legacyregistry" "k8s.io/component-base/metrics/testutil" kmsservice "k8s.io/kms/pkg/service" + "k8s.io/utils/pointer" ) const ( @@ -646,6 +648,584 @@ func TestKMSPluginHealthz(t *testing.T) { } } +// tests for masking rules +func TestWildcardMasking(t *testing.T) { + testCases := []struct { + desc string + config *apiserverconfig.EncryptionConfiguration + expectedError string + }{ + { + desc: "resources masked by *. group", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "*.", + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"", + }, + { + desc: "*. masked by *. group", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "*.", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms2", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"*.\" is masked by earlier rule \"*.\"", + }, + { + desc: "*.foo masked by *.foo", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "*.foo", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.foo", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms2", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"*.foo\" is masked by earlier rule \"*.foo\"", + }, + { + desc: "*.* masked by *.*", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "*.*", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.*", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms2", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"*.*\" is masked by earlier rule \"*.*\"", + }, + { + desc: "resources masked by *. group in multiple configurations", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.", + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "another-kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/another-testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"secrets\" is masked by earlier rule \"*.\"", + }, + { + desc: "resources masked by *.*", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "*.*", + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"", + }, + { + desc: "resources masked by *.* in multiple configurations", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.*", + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "another-kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/another-testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"secrets\" is masked by earlier rule \"*.*\"", + }, + { + desc: "resources *. masked by *.*", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "*.*", + "*.", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"", + }, + { + desc: "resources *. masked by *.* in multiple configurations", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "*.*", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "another-kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/another-testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + expectedError: "resource \"*.\" is masked by earlier rule \"*.*\"", + }, + { + desc: "resources not masked by any rule", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "secrets", + "*.*", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + }, + { + desc: "resources not masked by any rule in multiple configurations", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.*", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "another-kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/another-testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + _, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config) + if errString(err) != tc.expectedError { + t.Errorf("expected error %s but got %s", tc.expectedError, errString(err)) + } + }) + } +} + +func TestWildcardStructure(t *testing.T) { + testCases := []struct { + desc string + expectedResourceTransformers map[string]string + config *apiserverconfig.EncryptionConfiguration + errorValue string + }{ + { + desc: "should not result in error", + expectedResourceTransformers: map[string]string{ + "configmaps": "k8s:enc:kms:v1:kms:", + "secrets": "k8s:enc:kms:v1:another-kms:", + "events": "k8s:enc:kms:v1:fancy:", + "deployments.apps": "k8s:enc:kms:v1:kms:", + "pods": "k8s:enc:kms:v1:fancy:", + "pandas": "k8s:enc:kms:v1:fancy:", + "pandas.bears": "k8s:enc:kms:v1:yet-another-provider:", + "jobs.apps": "k8s:enc:kms:v1:kms:", + }, + + errorValue: "", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "*.apps", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "another-kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + { + Identity: &apiserverconfig.IdentityConfiguration{}, + }, + }, + }, + { + Resources: []string{ + "*.", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "fancy", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.*", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "yet-another-provider", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + }, + }, + }, + { + desc: "should result in error", + errorValue: "resource \"secrets\" is masked by earlier rule \"*.\"", + config: &apiserverconfig.EncryptionConfiguration{ + Resources: []apiserverconfig.ResourceConfiguration{ + { + Resources: []string{ + "configmaps", + "*.", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + }, + }, + { + Resources: []string{ + "*.*", + "secrets", + }, + Providers: []apiserverconfig.ProviderConfiguration{ + { + KMS: &apiserverconfig.KMSConfiguration{ + Name: "kms", + APIVersion: "v1", + Timeout: &metav1.Duration{Duration: 3 * time.Second}, + Endpoint: "unix:///tmp/testprovider.sock", + CacheSize: pointer.Int32(10), + }, + }, + { + Identity: &apiserverconfig.IdentityConfiguration{}, + }, + }, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + transformers, _, _, err := getTransformerOverridesAndKMSPluginProbes(ctx, tc.config) + if errString(err) != tc.errorValue { + t.Errorf("expected error %s but got %s", tc.errorValue, errString(err)) + } + + if len(tc.errorValue) > 0 { + return + } + + // check if expectedResourceTransformers are present + for resource, expectedTransformerName := range tc.expectedResourceTransformers { + transformer := transformerFromOverrides(transformers, schema.ParseGroupResource(resource)) + transformerName := string( + reflect.ValueOf(transformer).Elem().FieldByName("transformers").Index(0).FieldByName("Prefix").Bytes(), + ) + + if transformerName != expectedTransformerName { + t.Errorf("resource %s: expected same transformer name but got %v", resource, cmp.Diff(transformerName, expectedTransformerName)) + } + } + }) + } +} + func TestKMSPluginHealthzTTL(t *testing.T) { ctx := testContext(t) diff --git a/test/e2e/testing-manifests/auth/encrypt/encryption-config.yaml b/test/e2e/testing-manifests/auth/encrypt/encryption-config.yaml index eeb52ed2d7d1..b0bc7fe380db 100644 --- a/test/e2e/testing-manifests/auth/encrypt/encryption-config.yaml +++ b/test/e2e/testing-manifests/auth/encrypt/encryption-config.yaml @@ -1,64 +1,8 @@ apiVersion: apiserver.config.k8s.io/v1 kind: EncryptionConfiguration resources: - # The set of resources here are configured using output from "kubectl api-resources -o name" in a - # kind cluster running the latest built release. - resources: - - bindings - - componentstatuses - - configmaps - - endpoints - - events - - limitranges - - namespaces - - nodes - - persistentvolumeclaims - - persistentvolumes - - pods - - podtemplates - - replicationcontrollers - - resourcequotas - - secrets - - serviceaccounts - - services - - mutatingwebhookconfigurations.admissionregistration.k8s.io - - validatingwebhookconfigurations.admissionregistration.k8s.io - - customresourcedefinitions.apiextensions.k8s.io - - apiservices.apiregistration.k8s.io - - controllerrevisions.apps - - daemonsets.apps - - deployments.apps - - replicasets.apps - - statefulsets.apps - - tokenreviews.authentication.k8s.io - - localsubjectaccessreviews.authorization.k8s.io - - selfsubjectaccessreviews.authorization.k8s.io - - selfsubjectrulesreviews.authorization.k8s.io - - subjectaccessreviews.authorization.k8s.io - - horizontalpodautoscalers.autoscaling - - cronjobs.batch - - jobs.batch - - certificatesigningrequests.certificates.k8s.io - - leases.coordination.k8s.io - - endpointslices.discovery.k8s.io - - events.events.k8s.io - - flowschemas.flowcontrol.apiserver.k8s.io - - prioritylevelconfigurations.flowcontrol.apiserver.k8s.io - - ingressclasses.networking.k8s.io - - ingresses.networking.k8s.io - - networkpolicies.networking.k8s.io - - runtimeclasses.node.k8s.io - - poddisruptionbudgets.policy - - clusterrolebindings.rbac.authorization.k8s.io - - clusterroles.rbac.authorization.k8s.io - - rolebindings.rbac.authorization.k8s.io - - roles.rbac.authorization.k8s.io - - priorityclasses.scheduling.k8s.io - - csidrivers.storage.k8s.io - - csinodes.storage.k8s.io - - csistoragecapacities.storage.k8s.io - - storageclasses.storage.k8s.io - - volumeattachments.storage.k8s.io + - '*.*' providers: - kms: apiVersion: v2 diff --git a/test/integration/controlplane/transformation/kms_transformation_test.go b/test/integration/controlplane/transformation/kms_transformation_test.go index 27920aaf1c15..78d5b26d0476 100644 --- a/test/integration/controlplane/transformation/kms_transformation_test.go +++ b/test/integration/controlplane/transformation/kms_transformation_test.go @@ -34,13 +34,21 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" "golang.org/x/crypto/cryptobyte" + + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/apiserver/pkg/storage/value" aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes" mock "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/testing/v1beta1" + "k8s.io/apiserver/pkg/util/feature" + "k8s.io/client-go/dynamic" + featuregatetesting "k8s.io/component-base/featuregate/testing" kmsapi "k8s.io/kms/apis/v1beta1" + "k8s.io/kubernetes/test/integration" + "k8s.io/kubernetes/test/integration/etcd" ) const ( @@ -450,7 +458,6 @@ resources: } rawConfigmapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace)) - if err != nil { t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace), err) } @@ -533,6 +540,234 @@ resources: } } +func TestEncryptAll(t *testing.T) { + encryptionConfig := ` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - '*.*' + providers: + - kms: + name: encrypt-all-kms-provider + cachesize: 1000 + endpoint: unix:///@encrypt-all-kms-provider.sock +` + + t.Run("encrypt all resources", func(t *testing.T) { + pluginMock, err := mock.NewBase64Plugin("@encrypt-all-kms-provider.sock") + if err != nil { + t.Fatalf("failed to create mock of KMS Plugin: %v", err) + } + + go pluginMock.Start() + if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil { + t.Fatalf("Failed start plugin, err: %v", err) + } + defer pluginMock.CleanUp() + + defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllAlpha", true)() + defer featuregatetesting.SetFeatureGateDuringTest(t, feature.DefaultFeatureGate, "AllBeta", true)() + test, err := newTransformTest(t, encryptionConfig, false, "") + if err != nil { + t.Fatalf("failed to start KUBE API Server with encryptionConfig") + } + defer test.cleanUp() + + _, serverResources, err := test.restClient.Discovery().ServerGroupsAndResources() + if err != nil { + t.Fatal(err) + } + resources := etcd.GetResources(t, serverResources) + client := dynamic.NewForConfigOrDie(test.kubeAPIServer.ClientConfig) + + etcdStorageData := etcd.GetEtcdStorageDataForNamespace(testNamespace) + for _, resource := range resources { + gvr := resource.Mapping.Resource + stub := etcdStorageData[gvr].Stub + + // continue if stub is empty + if stub == "" { + t.Errorf("skipping resource %s because stub is empty", gvr) + continue + } + + dynamicClient, obj, err := etcd.JSONToUnstructured(stub, testNamespace, &meta.RESTMapping{ + Resource: gvr, + GroupVersionKind: gvr.GroupVersion().WithKind(resource.Mapping.GroupVersionKind.Kind), + Scope: resource.Mapping.Scope, + }, client) + if err != nil { + t.Fatal(err) + } + + _, err = dynamicClient.Create(context.TODO(), obj, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + } + + rawClient, etcdClient, err := integration.GetEtcdClients(test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Transport) + if err != nil { + t.Fatalf("failed to create etcd client: %v", err) + } + // kvClient is a wrapper around rawClient and to avoid leaking goroutines we need to + // close the client (which we can do by closing rawClient). + defer rawClient.Close() + + response, err := etcdClient.Get(context.TODO(), "/"+test.kubeAPIServer.ServerOpts.Etcd.StorageConfig.Prefix, clientv3.WithPrefix()) + if err != nil { + t.Fatalf("failed to retrieve secret from etcd %v", err) + } + + // assert that total key values in response in greater than 0 + if len(response.Kvs) == 0 { + t.Fatalf("expected total number of keys to be greater than 0, but got %d", len(response.Kvs)) + } + + // assert that total response keys are greater or equal to total resources + if len(response.Kvs) < len(resources) { + t.Fatalf("expected total number of keys to be greater or equal to total resources, but got %d", len(response.Kvs)) + } + + wantPrefix := "k8s:enc:kms:v1:encrypt-all-kms-provider:" + for _, kv := range response.Kvs { + // the following resources are not encrypted as they are not REST APIs and hence are not expected + // to be encrypted because it would be impossible to perform a storage migration on them + if strings.Contains(kv.String(), "masterleases") || + strings.Contains(kv.String(), "serviceips") || + strings.Contains(kv.String(), "servicenodeports") { + // assert that these resources are not encrypted with any provider + if bytes.HasPrefix(kv.Value, []byte("k8s:enc:")) { + t.Errorf("expected resource %s to not be prefixed with %s, but got %s", kv.Key, "k8s:enc:", kv.Value) + } + continue + } + + // assert that all other resources are encrypted + if !bytes.HasPrefix(kv.Value, []byte(wantPrefix)) { + t.Errorf("expected resource %s to be prefixed with %s, but got %s", kv.Key, wantPrefix, kv.Value) + } + } + }) +} + +func TestEncryptAllWithWildcard(t *testing.T) { + encryptionConfig := ` +kind: EncryptionConfiguration +apiVersion: apiserver.config.k8s.io/v1 +resources: + - resources: + - configmaps + providers: + - identity: {} + - resources: + - '*.batch' + providers: + - kms: + name: kms-provider + cachesize: 1000 + endpoint: unix:///@kms-provider.sock + - resources: + - '*.*' + providers: + - kms: + name: encrypt-all-kms-provider + cachesize: 1000 + endpoint: unix:///@encrypt-all-kms-provider.sock +` + pluginMock, err := mock.NewBase64Plugin("@kms-provider.sock") + if err != nil { + t.Fatalf("failed to create mock of KMS Plugin: %v", err) + } + + go pluginMock.Start() + if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil { + t.Fatalf("Failed start plugin, err: %v", err) + } + defer pluginMock.CleanUp() + + encryptAllPluginMock, err := mock.NewBase64Plugin("@encrypt-all-kms-provider.sock") + if err != nil { + t.Fatalf("failed to create mock of KMS Plugin: %v", err) + } + + go encryptAllPluginMock.Start() + if err := mock.WaitForBase64PluginToBeUp(pluginMock); err != nil { + t.Fatalf("Failed start plugin, err: %v", err) + } + defer encryptAllPluginMock.CleanUp() + + test, err := newTransformTest(t, encryptionConfig, false, "") + if err != nil { + t.Fatalf("failed to start KUBE API Server with encryptionConfig\n %s, error: %v", encryptionConfig, err) + } + defer test.cleanUp() + + wantPrefix := "k8s:enc:kms:v1:kms-provider:" + wantPrefixForEncryptAll := "k8s:enc:kms:v1:encrypt-all-kms-provider:" + + _, err = test.createJob("test-job", "default") + if err != nil { + t.Fatalf("failed to create job: %v", err) + } + + rawJobsEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "jobs", "test-job", "default")) + if err != nil { + t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "jobs", "test-job", "default"), err) + } + + // assert prefix for jobs + if !bytes.HasPrefix(rawJobsEnvelope.Kvs[0].Value, []byte(wantPrefix)) { + t.Fatalf("expected jobs to be prefixed with %s, but got %s", wantPrefix, rawJobsEnvelope.Kvs[0].Value) + } + + _, err = test.createDeployment("test-deployment", "default") + if err != nil { + t.Fatalf("failed to create deployment: %v", err) + } + + rawDeploymentsEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", "test-deployment", "default")) + if err != nil { + t.Fatalf("failed to read %s from etcd: %v", test.getETCDPathForResource(test.storageConfig.Prefix, "", "deployments", "test-deployment", "default"), err) + } + + // assert prefix for deployments + if !bytes.HasPrefix(rawDeploymentsEnvelope.Kvs[0].Value, []byte(wantPrefixForEncryptAll)) { + t.Fatalf("expected deployments to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawDeploymentsEnvelope.Kvs[0].Value) + } + + test.secret, err = test.createSecret(testSecret, testNamespace) + if err != nil { + t.Fatalf("Failed to create test secret, error: %v", err) + } + + rawSecretEnvelope, err := test.getRawSecretFromETCD() + if err != nil { + t.Fatalf("failed to read secrets from etcd: %v", err) + } + + // assert prefix for secrets + if !bytes.HasPrefix(rawSecretEnvelope, []byte(wantPrefixForEncryptAll)) { + t.Fatalf("expected secrets to be prefixed with %s, but got %s", wantPrefixForEncryptAll, rawSecretEnvelope) + } + + _, err = test.createConfigMap(testConfigmap, testNamespace) + if err != nil { + t.Fatalf("Failed to create test configmap, error: %v", err) + } + + rawConfigMapEnvelope, err := test.readRawRecordFromETCD(test.getETCDPathForResource(test.storageConfig.Prefix, "", "configmaps", testConfigmap, testNamespace)) + if err != nil { + t.Fatalf("failed to read configmaps from etcd: %v", err) + } + + // assert configmaps do not have the encrypted data prefix + if bytes.HasPrefix(rawConfigMapEnvelope.Kvs[0].Value, []byte("k8s:enc:")) { + t.Fatalf("expected configmaps to be not encrypted, got %s", rawConfigMapEnvelope.Kvs[0].Value) + } +} + func TestEncryptionConfigHotReloadFileWatch(t *testing.T) { testCases := []struct { sleep time.Duration diff --git a/test/integration/controlplane/transformation/transformation_test.go b/test/integration/controlplane/transformation/transformation_test.go index b090cfae1338..407945576cf3 100644 --- a/test/integration/controlplane/transformation/transformation_test.go +++ b/test/integration/controlplane/transformation/transformation_test.go @@ -30,6 +30,8 @@ import ( clientv3 "go.etcd.io/etcd/client/v3" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -47,6 +49,7 @@ import ( "k8s.io/kubernetes/test/integration" "k8s.io/kubernetes/test/integration/etcd" "k8s.io/kubernetes/test/integration/framework" + "k8s.io/utils/pointer" "sigs.k8s.io/yaml" ) @@ -337,6 +340,79 @@ func (e *transformTest) createConfigMap(name, namespace string) (*corev1.ConfigM return cm, nil } +// create jobs +func (e *transformTest) createJob(name, namespace string) (*batchv1.Job, error) { + job := &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: batchv1.JobSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "test", + }, + }, + RestartPolicy: corev1.RestartPolicyNever, + }, + }, + }, + } + if _, err := e.restClient.BatchV1().Jobs(job.Namespace).Create(context.TODO(), job, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("error while creating job: %v", err) + } + + return job, nil +} + +// create deployment +func (e *transformTest) createDeployment(name, namespace string) (*appsv1.Deployment, error) { + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: pointer.Int32(2), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "nginx", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "nginx", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "nginx", + Image: "nginx:1.17", + Ports: []corev1.ContainerPort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + ContainerPort: 80, + }, + }, + }, + }, + }, + }, + }, + } + if _, err := e.restClient.AppsV1().Deployments(deployment.Namespace).Create(context.TODO(), deployment, metav1.CreateOptions{}); err != nil { + return nil, fmt.Errorf("error while creating deployment: %v", err) + } + + return deployment, nil +} + func gvr(group, version, resource string) schema.GroupVersionResource { return schema.GroupVersionResource{Group: group, Version: version, Resource: resource} }