feat(vault) implement controller and translator (#5384)

Add controller and translator for KongVault resources.

CHANGELOG.md

@@ -129,9 +129,13 @@ Adding a new version? You'll need three changes:
 - Added functionality to the `KongUpstreamPolicy` controller to properly set and
   enforce `KongUpstreamPolicy` status.
   [#5185](https://github.com/Kong/kubernetes-ingress-controller/pull/5185)
-- Add CRD `KongVault` to reperesent a custom Kong vault for storing senstive
-  data used in plugin configurations.
+- New CRD `KongVault` to reperesent a custom Kong vault for storing senstive
+  data used in plugin configurations. Now users can create a `KongVault` to
+  create a custom Kong vault and reference the values in configurations of
+  plugins. Reference of using Kong vaults: [Kong vault]
   [#5354](https://github.com/Kong/kubernetes-ingress-controller/pull/5354)
+  [#5384](https://github.com/Kong/kubernetes-ingress-controller/pull/5384)
+
 
 ### Fixed
 
@@ -165,6 +169,8 @@ Adding a new version? You'll need three changes:
   - `KongPlugin` and `KongClusterPlugin` now enforce `plugin` to be immutable.
     [#5142](https://github.com/Kong/kubernetes-ingress-controller/pull/5142)
 
+[Kong vault]: https://docs.konghq.com/gateway/latest/kong-enterprise/secrets-management/
+
 ## [3.0.1]
 
 > Release date: 2023-11-22

config/components/manager_dev_webhook/validating_webhook_configuration.yaml

@@ -23,9 +23,11 @@ webhooks:
     - UPDATE
     resources:
     - kongconsumers
+    - kongconsumergroups
     - kongplugins
     - kongclusterplugins
     - kongingresses
+    - kongvaults
     scope: '*'
   - apiGroups:
     - ""

config/crd/kustomization.yaml

@@ -13,6 +13,7 @@ resources:
 - bases/configuration.konghq.com_kongplugins.yaml
 - bases/configuration.konghq.com_ingressclassparameterses.yaml
 - bases/configuration.konghq.com_kongupstreampolicies.yaml
+- bases/configuration.konghq.com_kongvaults.yaml
 #+kubebuilder:scaffold:crdkustomizeresource
 
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.

config/rbac/role.yaml

@@ -153,6 +153,22 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - configuration.konghq.com
+  resources:
+  - kongvaults
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - configuration.konghq.com
+  resources:
+  - kongvaults/status
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - configuration.konghq.com
   resources:

docs/cli-arguments.md

@@ -27,6 +27,7 @@
 | `--enable-controller-ingress-networkingv1` | `bool` | Enable the networking.k8s.io/v1 Ingress controller. | `true` |
 | `--enable-controller-kong-service-facade` | `bool` | Enable the KongServiceFacade controller. | `true` |
 | `--enable-controller-kong-upstream-policy` | `bool` | Enable the KongUpstreamPolicy controller. | `true` |
+| `--enable-controller-kong-vault` | `bool` | Enable the KongVault controller. | `true` |
 | `--enable-controller-kongclusterplugin` | `bool` | Enable the KongClusterPlugin controller. | `true` |
 | `--enable-controller-kongconsumer` | `bool` | Enable the KongConsumer controller. | `true` |
 | `--enable-controller-kongingress` | `bool` | Enable the KongIngress controller. | `true` |

hack/deploy-admission-controller.sh

@@ -59,6 +59,7 @@ webhooks:
     - kongplugins
     - kongclusterplugins
     - kongingresses
+    - kongvaults
   - apiGroups:
     - ''
     apiVersions:

hack/generators/controllers/networking/main.go

@@ -247,6 +247,23 @@ var inputControllersNeeded = &typesNeeded{
 		AcceptsIngressClassNameAnnotation: true,
 		RBACVerbs:                         []string{"get", "list", "watch"},
 	},
+	typeNeeded{
+		Group:                            "configuration.konghq.com",
+		Version:                          "v1alpha1",
+		Kind:                             "KongVault",
+		PackageImportAlias:               "kongv1alpha1",
+		PackageAlias:                     "KongV1Alpha1",
+		Package:                          kongv1alpha1,
+		Plural:                           "kongvaults",
+		CacheType:                        "KongVault",
+		NeedsStatusPermissions:           true,
+		ConfigStatusNotificationsEnabled: true,
+		ProgrammedCondition: ProgrammedConditionConfiguration{
+			UpdatesEnabled: true,
+		},
+		AcceptsIngressClassNameAnnotation: true,
+		RBACVerbs:                         []string{"get", "list", "watch"},
+	},
 }
 
 var inputRBACPermissionsNeeded = &rbacsNeeded{

internal/admission/adminapi_provider.go

@@ -61,6 +61,14 @@ func (p DefaultAdminAPIServicesProvider) GetRoutesService() (kong.AbstractRouteS
 	return c.Routes, true
 }
 
+func (p DefaultAdminAPIServicesProvider) GetVaultsService() (kong.AbstractVaultService, bool) {
+	c, ok := p.designatedAdminAPIClient()
+	if !ok {
+		return nil, ok
+	}
+	return c.Vaults, true
+}
+
 func (p DefaultAdminAPIServicesProvider) designatedAdminAPIClient() (*kong.Client, bool) {
 	gwClients := p.gatewayClientsProvider.GatewayClients()
 	if len(gwClients) == 0 {

internal/admission/errors.go

@@ -14,6 +14,9 @@ const (
 	ErrTextPluginConfigValidationFailed       = "unable to validate plugin schema"
 	ErrTextPluginConfigViolatesSchema         = "plugin failed schema validation: %s"
 	ErrTextPluginSecretConfigUnretrievable    = "could not load secret plugin configuration"
+	ErrTextVaultConfigUnmarshalFailed         = "failed to unmarshal vault configuration: %v"
+	ErrTextVaultUnableToValidate              = "unable to validate vault on Kong gateway"
+	ErrTextVaultConfigValidationResultInvalid = "vault configuration in invalid: %s"
 )
 
 const (

internal/admission/handler.go

@@ -17,6 +17,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1"
 )
 
@@ -96,6 +97,11 @@ var (
 		Version:  kongv1.SchemeGroupVersion.Version,
 		Resource: "kongingresses",
 	}
+	kongVaultGVResource = metav1.GroupVersionResource{
+		Group:    kongv1alpha1.SchemeGroupVersion.Group,
+		Version:  kongv1alpha1.SchemeGroupVersion.Version,
+		Resource: "kongvaults",
+	}
 	secretGVResource = metav1.GroupVersionResource{
 		Group:    corev1.SchemeGroupVersion.Group,
 		Version:  corev1.SchemeGroupVersion.Version,
@@ -135,6 +141,8 @@ func (h RequestHandler) handleValidation(ctx context.Context, request admissionv
 		return h.handleHTTPRoute(ctx, request, responseBuilder)
 	case kongIngressGVResource:
 		return h.handleKongIngress(ctx, request, responseBuilder)
+	case kongVaultGVResource:
+		return h.handleKongVault(ctx, request, responseBuilder)
 	case serviceGVResource:
 		return h.handleService(ctx, request, responseBuilder)
 	case ingressGVResource:
@@ -426,3 +434,17 @@ func (h RequestHandler) handleIngress(ctx context.Context, request admissionv1.A
 
 	return responseBuilder.Allowed(ok).WithMessage(message).Build(), nil
 }
+
+func (h RequestHandler) handleKongVault(ctx context.Context, request admissionv1.AdmissionRequest, responseBuilder *ResponseBuilder) (*admissionv1.AdmissionResponse, error) {
+	kongVault := kongv1alpha1.KongVault{}
+	_, _, err := codecs.UniversalDeserializer().Decode(request.Object.Raw, nil, &kongVault)
+	if err != nil {
+		return nil, err
+	}
+	ok, message, err := h.Validator.ValidateVault(ctx, kongVault)
+	if err != nil {
+		return nil, err
+	}
+
+	return responseBuilder.Allowed(ok).WithMessage(message).Build(), nil
+}

internal/admission/server_test.go

@@ -20,6 +20,7 @@ import (
 
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/gatewayapi"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1"
 )
 
@@ -77,6 +78,10 @@ func (v KongFakeValidator) ValidateIngress(_ context.Context, _ netv1.Ingress) (
 	return v.Result, v.Message, v.Error
 }
 
+func (v KongFakeValidator) ValidateVault(_ context.Context, _ kongv1alpha1.KongVault) (bool, string, error) {
+	return v.Result, v.Message, v.Error
+}
+
 func TestServeHTTPBasic(t *testing.T) {
 	assert := assert.New(t)
 	res := httptest.NewRecorder()

internal/admission/validator.go

@@ -26,6 +26,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1"
 )
 
@@ -35,6 +36,7 @@ type KongValidator interface {
 	ValidateConsumerGroup(ctx context.Context, consumerGroup kongv1beta1.KongConsumerGroup) (bool, string, error)
 	ValidatePlugin(ctx context.Context, plugin kongv1.KongPlugin, overrideSecrets []*corev1.Secret) (bool, string, error)
 	ValidateClusterPlugin(ctx context.Context, plugin kongv1.KongClusterPlugin, overrideSecrets []*corev1.Secret) (bool, string, error)
+	ValidateVault(ctx context.Context, vault kongv1alpha1.KongVault) (bool, string, error)
 	ValidateCredential(ctx context.Context, secret corev1.Secret) (bool, string)
 	ValidateGateway(ctx context.Context, gateway gatewayapi.Gateway) (bool, string, error)
 	ValidateHTTPRoute(ctx context.Context, httproute gatewayapi.HTTPRoute) (bool, string, error)
@@ -49,6 +51,7 @@ type AdminAPIServicesProvider interface {
 	GetConsumerGroupsService() (kong.AbstractConsumerGroupService, bool)
 	GetInfoService() (kong.AbstractInfoService, bool)
 	GetRoutesService() (kong.AbstractRouteService, bool)
+	GetVaultsService() (kong.AbstractVaultService, bool)
 }
 
 // ConsumerGetter is an interface for retrieving KongConsumers.
@@ -520,6 +523,30 @@ func (noOpRoutesValidator) Validate(_ context.Context, _ *kong.Route) (bool, str
 	return true, "", nil
 }
 
+func (validator KongHTTPValidator) ValidateVault(ctx context.Context, k8sKongVault kongv1alpha1.KongVault) (bool, string, error) {
+	// Ignore KongVaults that are being managed by another controller.
+	if !validator.ingressClassMatcher(&k8sKongVault.ObjectMeta, annotations.IngressClassKey, annotations.ExactClassMatch) {
+		return true, "", nil
+	}
+	config, err := kongstate.RawConfigToConfiguration(k8sKongVault.Spec.Config.Raw)
+	if err != nil {
+		return false, fmt.Sprintf(ErrTextVaultConfigUnmarshalFailed, err), nil
+	}
+	kongVault := kong.Vault{
+		Name:        kong.String(k8sKongVault.Spec.Backend),
+		Prefix:      kong.String(k8sKongVault.Spec.Prefix),
+		Description: kong.String(k8sKongVault.Spec.Description),
+		Config:      config,
+	}
+	// TODO: /schemas/vaults/test does not check "unique" restraint on `prefix` field:
+	// https://github.com/Kong/kubernetes-ingress-controller/issues/5395
+	errText, err := validator.validateVaultAgainstGatewaySchema(ctx, kongVault)
+	if err != nil || errText != "" {
+		return false, errText, err
+	}
+	return true, "", nil
+}
+
 // -----------------------------------------------------------------------------
 // KongHTTPValidator - Private Methods
 // -----------------------------------------------------------------------------
@@ -582,6 +609,21 @@ func (validator KongHTTPValidator) validatePluginAgainstGatewaySchema(ctx contex
 	return "", nil
 }
 
+func (validator KongHTTPValidator) validateVaultAgainstGatewaySchema(ctx context.Context, vault kong.Vault) (string, error) {
+	vaultService, hasClient := validator.AdminAPIServicesProvider.GetVaultsService()
+	if !hasClient {
+		return "", nil
+	}
+	isValid, msg, err := vaultService.Validate(ctx, &vault)
+	if err != nil {
+		return ErrTextVaultUnableToValidate, err
+	}
+	if !isValid {
+		return fmt.Sprintf(ErrTextVaultConfigValidationResultInvalid, msg), nil
+	}
+	return "", nil
+}
+
 type managerClientSecretGetter struct {
 	managerClient client.Client
 }

internal/admission/validator_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/store"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util/builder"
 	kongv1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1"
+	kongv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1alpha1"
 	kongv1beta1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/configuration/v1beta1"
 	incubatorv1alpha1 "github.com/kong/kubernetes-ingress-controller/v3/pkg/apis/incubator/v1alpha1"
 )
@@ -62,6 +63,7 @@ type fakeServicesProvider struct {
 	consumerGroupSvc kong.AbstractConsumerGroupService
 	infoSvc          kong.AbstractInfoService
 	routeSvc         kong.AbstractRouteService
+	vaultSvc         kong.AbstractVaultService
 }
 
 func (f fakeServicesProvider) GetConsumersService() (kong.AbstractConsumerService, bool) {
@@ -99,6 +101,13 @@ func (f fakeServicesProvider) GetRoutesService() (kong.AbstractRouteService, boo
 	return nil, false
 }
 
+func (f fakeServicesProvider) GetVaultsService() (kong.AbstractVaultService, bool) {
+	if f.vaultSvc != nil {
+		return f.vaultSvc, true
+	}
+	return nil, false
+}
+
 func TestKongHTTPValidator_ValidatePlugin(t *testing.T) {
 	store, _ := store.NewFakeStore(store.FakeObjects{
 		Secrets: []*corev1.Secret{
@@ -1227,3 +1236,90 @@ func (f *fakeRouteSvc) Validate(context.Context, *kong.Route) (bool, string, err
 	}
 	return true, "", nil
 }
+
+func TestValidator_ValidateVault(t *testing.T) {
+	testCases := []struct {
+		name            string
+		kongVault       kongv1alpha1.KongVault
+		validateSvcFail bool
+		expectedOK      bool
+		expectedMessage string
+		expectedError   string
+	}{
+		{
+			name: "valid vault",
+			kongVault: kongv1alpha1.KongVault{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "vault-1",
+				},
+				Spec: kongv1alpha1.KongVaultSpec{
+					Backend: "env",
+					Prefix:  "env-1",
+				},
+			},
+			expectedOK: true,
+		},
+		{
+			name: "vault with invalid(malformed) configuration",
+			kongVault: kongv1alpha1.KongVault{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "vault-1",
+				},
+				Spec: kongv1alpha1.KongVaultSpec{
+					Backend: "env",
+					Prefix:  "env-1",
+					Config: apiextensionsv1.JSON{
+						Raw: []byte(`{{}`),
+					},
+				},
+			},
+			expectedOK:      false,
+			expectedMessage: "failed to unmarshal vault configuration",
+		},
+		{
+			name: "vault with failure in validating service",
+			kongVault: kongv1alpha1.KongVault{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "vault-1",
+				},
+				Spec: kongv1alpha1.KongVaultSpec{
+					Backend: "env",
+					Prefix:  "env-1",
+				},
+			},
+			validateSvcFail: true,
+			expectedOK:      false,
+			expectedMessage: "something is wrong with the vault",
+		},
+	}
+	for _, tc := range testCases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			validator := KongHTTPValidator{
+				AdminAPIServicesProvider: fakeServicesProvider{
+					vaultSvc: &fakeVaultSvc{
+						shouldFail: tc.validateSvcFail,
+					},
+				},
+				ingressClassMatcher: fakeClassMatcher,
+				Logger:              logr.Discard(),
+			}
+			ok, msg, err := validator.ValidateVault(context.Background(), tc.kongVault)
+			require.NoError(t, err)
+			assert.Equal(t, tc.expectedOK, ok)
+			assert.Contains(t, msg, tc.expectedMessage)
+		})
+	}
+}
+
+type fakeVaultSvc struct {
+	kong.AbstractVaultService
+	shouldFail bool
+}
+
+func (s fakeVaultSvc) Validate(context.Context, *kong.Vault) (bool, string, error) {
+	if s.shouldFail {
+		return false, "something is wrong with the vault", nil
+	}
+	return true, "", nil
+}