From db23878d3fb92e09eb1906fc8e3f767e3ff721ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Patryk=20Ma=C5=82ek?= Date: Tue, 21 Mar 2023 20:34:31 +0100 Subject: [PATCH] tests: add envtests unit test for HTTPRouteReconciler --- .../kongadminapi_controller_envtest_test.go | 4 +- internal/controllers/controller.go | 11 + .../controllers/gateway/dataplane_client.go | 26 ++ .../gateway/dataplane_mock_test.go | 32 ++ internal/controllers/gateway/gateway_utils.go | 3 +- .../gateway/httproute_controller.go | 8 +- .../httproute_controller_envtest_test.go | 350 ++++++++++++++++++ internal/controllers/gateway/route_utils.go | 16 +- internal/util/builder/httpbackendref.go | 4 + test/envtest/controller.go | 53 +++ test/envtest/k8s.go | 4 + test/envtest/setup.go | 86 ++++- test/helpers/conditions.go | 66 ++++ 13 files changed, 654 insertions(+), 9 deletions(-) create mode 100644 internal/controllers/controller.go create mode 100644 internal/controllers/gateway/dataplane_client.go create mode 100644 internal/controllers/gateway/dataplane_mock_test.go create mode 100644 internal/controllers/gateway/httproute_controller_envtest_test.go create mode 100644 test/envtest/controller.go create mode 100644 test/helpers/conditions.go diff --git a/internal/controllers/configuration/kongadminapi_controller_envtest_test.go b/internal/controllers/configuration/kongadminapi_controller_envtest_test.go index 1a55db5ddd..b409e75581 100644 --- a/internal/controllers/configuration/kongadminapi_controller_envtest_test.go +++ b/internal/controllers/configuration/kongadminapi_controller_envtest_test.go @@ -60,7 +60,7 @@ func startKongAdminAPIServiceReconciler(ctx context.Context, t *testing.T, clien mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Logger: logrusr.New(logrus.New()), - Scheme: scheme.Scheme, + Scheme: client.Scheme(), SyncPeriod: lo.ToPtr(2 * time.Second), MetricsBindAddress: "0", }) @@ -106,7 +106,7 @@ func TestKongAdminAPIController(t *testing.T) { // In tests below we use a deferred cancel to stop the manager and not wait // for its timeout. - cfg := envtest.Setup(t) + cfg := envtest.Setup(t, scheme.Scheme) client, err := ctrlclient.New(cfg, ctrlclient.Options{}) require.NoError(t, err) diff --git a/internal/controllers/controller.go b/internal/controllers/controller.go new file mode 100644 index 0000000000..b1acf57911 --- /dev/null +++ b/internal/controllers/controller.go @@ -0,0 +1,11 @@ +package controllers + +import ( + "github.com/go-logr/logr" + ctrl "sigs.k8s.io/controller-runtime" +) + +type Reconciler interface { + SetupWithManager(ctrl.Manager) error + SetLogger(logr.Logger) +} diff --git a/internal/controllers/gateway/dataplane_client.go b/internal/controllers/gateway/dataplane_client.go new file mode 100644 index 0000000000..be06870683 --- /dev/null +++ b/internal/controllers/gateway/dataplane_client.go @@ -0,0 +1,26 @@ +package gateway + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object" +) + +// DataPlane is a common interface that is used by reconcilers to interact +// with the dataplane. +// +// TODO: This can probably be used in other reconcilers as well. +// Related issue: https://github.com/Kong/kubernetes-ingress-controller/issues/3794 +type DataPlane interface { + DataPlaneClient + + AreKubernetesObjectReportsEnabled() bool + KubernetesObjectConfigurationStatus(obj client.Object) k8sobj.ConfigurationStatus +} + +// DataPlaneClient is a common client interface that is used by reconcilers to interact +// with the dataplane to perform CRUD operations on provided objects. +type DataPlaneClient interface { + UpdateObject(obj client.Object) error + DeleteObject(obj client.Object) error +} diff --git a/internal/controllers/gateway/dataplane_mock_test.go b/internal/controllers/gateway/dataplane_mock_test.go new file mode 100644 index 0000000000..33128889d6 --- /dev/null +++ b/internal/controllers/gateway/dataplane_mock_test.go @@ -0,0 +1,32 @@ +package gateway + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object" +) + +type DataplaneMock struct { + KubernetesObjectReportsEnabled bool + // Mapping namespace to name to status + // Note: this will come in useful when implementing + // https://github.com/Kong/kubernetes-ingress-controller/issues/3793 + // which requires the status to be reported for route objects. + ObjectsStatuses map[string]map[string]k8sobj.ConfigurationStatus +} + +func (d DataplaneMock) UpdateObject(_ client.Object) error { + return nil +} + +func (d DataplaneMock) DeleteObject(_ client.Object) error { + return nil +} + +func (d DataplaneMock) AreKubernetesObjectReportsEnabled() bool { + return d.KubernetesObjectReportsEnabled +} + +func (d DataplaneMock) KubernetesObjectConfigurationStatus(obj client.Object) k8sobj.ConfigurationStatus { + return d.ObjectsStatuses[obj.GetNamespace()][obj.GetName()] +} diff --git a/internal/controllers/gateway/gateway_utils.go b/internal/controllers/gateway/gateway_utils.go index 201b5b426a..88b1794c40 100644 --- a/internal/controllers/gateway/gateway_utils.go +++ b/internal/controllers/gateway/gateway_utils.go @@ -83,7 +83,8 @@ func isObjectUnmanaged(anns map[string]string) bool { // isGatewayClassControlledAndUnmanaged returns boolean if the GatewayClass // is controlled by this controller and is configured for unmanaged mode. func isGatewayClassControlledAndUnmanaged(gatewayClass *GatewayClass) bool { - return gatewayClass.Spec.ControllerName == GetControllerName() && isObjectUnmanaged(gatewayClass.Annotations) + isUnamanaged := isObjectUnmanaged(gatewayClass.Annotations) + return gatewayClass.Spec.ControllerName == GetControllerName() && isUnamanaged } // getRefFromPublishService splits a publish service string in the format namespace/name into a types.NamespacedName diff --git a/internal/controllers/gateway/httproute_controller.go b/internal/controllers/gateway/httproute_controller.go index e2225999fe..f594878be0 100644 --- a/internal/controllers/gateway/httproute_controller.go +++ b/internal/controllers/gateway/httproute_controller.go @@ -23,7 +23,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "github.com/kong/kubernetes-ingress-controller/v2/internal/dataplane" "github.com/kong/kubernetes-ingress-controller/v2/internal/util" k8sobj "github.com/kong/kubernetes-ingress-controller/v2/internal/util/kubernetes/object" ) @@ -38,7 +37,7 @@ type HTTPRouteReconciler struct { Log logr.Logger Scheme *runtime.Scheme - DataplaneClient *dataplane.KongClient + DataplaneClient DataPlane // If EnableReferenceGrant is true, we will check for ReferenceGrant if backend in another // namespace is in backendRefs. // If it is false, referencing backend in different namespace will be rejected. @@ -692,3 +691,8 @@ func (r *HTTPRouteReconciler) getHTTPRouteRuleReason(ctx context.Context, httpRo } return gatewayv1beta1.RouteReasonResolvedRefs, nil } + +// SetLogger sets the logger. +func (r *HTTPRouteReconciler) SetLogger(l logr.Logger) { + r.Log = l +} diff --git a/internal/controllers/gateway/httproute_controller_envtest_test.go b/internal/controllers/gateway/httproute_controller_envtest_test.go new file mode 100644 index 0000000000..813d0795c0 --- /dev/null +++ b/internal/controllers/gateway/httproute_controller_envtest_test.go @@ -0,0 +1,350 @@ +//go:build envtest +// +build envtest + +package gateway_test + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes/scheme" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway" + "github.com/kong/kubernetes-ingress-controller/v2/internal/util/builder" + "github.com/kong/kubernetes-ingress-controller/v2/test/envtest" + "github.com/kong/kubernetes-ingress-controller/v2/test/helpers" +) + +func init() { + if err := gatewayv1beta1.Install(scheme.Scheme); err != nil { + panic(err) + } +} + +func TestHTTPRouteReconcilerProperlyReactsToReferenceGrant(t *testing.T) { + t.Parallel() + + const ( + waitDuration = 5 * time.Second + tickDuration = 100 * time.Millisecond + ) + + cfg := envtest.Setup(t, scheme.Scheme) + var client ctrlclient.Client + { + var err error + client, err = ctrlclient.New(cfg, ctrlclient.Options{ + Scheme: scheme.Scheme, + }) + require.NoError(t, err) + } + + // In tests below we use a deferred cancel to stop the manager and not wait + // for its timeout. + + testcases := []struct { + name string + reconciler *gateway.HTTPRouteReconciler + }{ + { + name: "with ReferenceGrant enabled and with KubernetesObjectsReportsEnabled enabled", + reconciler: &gateway.HTTPRouteReconciler{ + DataplaneClient: func() gateway.DataPlane { + return gateway.DataplaneMock{ + KubernetesObjectReportsEnabled: true, + } + }(), + Client: client, + EnableReferenceGrant: true, + }, + }, + { + name: "with ReferenceGrant enabled and with KubernetesObjectsReportsEnabled disabled", + reconciler: &gateway.HTTPRouteReconciler{ + DataplaneClient: func() gateway.DataPlane { + return gateway.DataplaneMock{ + KubernetesObjectReportsEnabled: false, + } + }(), + Client: client, + EnableReferenceGrant: true, + }, + }, + } + + for _, tc := range testcases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + ns := envtest.CreateNamespace(ctx, t, client) + nsRoute := envtest.CreateNamespace(ctx, t, client) + + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: "backend-1", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "http", + Protocol: corev1.ProtocolTCP, + Port: 80, + TargetPort: intstr.FromInt(80), + }, + }, + }, + } + require.NoError(t, client.Create(ctx, &svc)) + + envtest.StartReconciler(ctx, t, client.Scheme(), cfg, tc.reconciler, nil) + + gwc := gatewayv1beta1.GatewayClass{ + Spec: gatewayv1beta1.GatewayClassSpec{ + ControllerName: gateway.GetControllerName(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: uuid.NewString(), + Annotations: map[string]string{ + "konghq.com/gatewayclass-unmanaged": "placeholder", + }, + }, + } + require.NoError(t, client.Create(ctx, &gwc)) + t.Cleanup(func() { _ = client.Delete(ctx, &gwc) }) + + gw := gatewayv1beta1.Gateway{ + Spec: gatewayv1beta1.GatewaySpec{ + GatewayClassName: gatewayv1beta1.ObjectName(gwc.Name), + Listeners: []gatewayv1beta1.Listener{ + { + Name: gatewayv1beta1.SectionName("http"), + Port: gatewayv1beta1.PortNumber(80), + Protocol: gatewayv1beta1.HTTPProtocolType, + AllowedRoutes: &gatewayv1beta1.AllowedRoutes{ + Namespaces: &gatewayv1beta1.RouteNamespaces{ + From: lo.ToPtr(gatewayv1beta1.NamespacesFromAll), + }, + }, + }, + }, + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: uuid.NewString(), + }, + } + require.NoError(t, client.Create(ctx, &gw)) + + gwOld := gw.DeepCopy() + gw.Status = gatewayv1beta1.GatewayStatus{ + Addresses: []gatewayv1beta1.GatewayAddress{ + { + Type: lo.ToPtr(gatewayv1beta1.IPAddressType), + Value: "10.0.0.1", + }, + }, + Conditions: []metav1.Condition{ + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "Ready", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + { + Type: "Programmed", + Status: metav1.ConditionTrue, + Reason: "Programmed", + LastTransitionTime: metav1.Now(), + ObservedGeneration: gw.Generation, + }, + }, + Listeners: []gatewayv1beta1.ListenerStatus{ + { + Name: gatewayv1beta1.SectionName("http"), + Conditions: []metav1.Condition{ + { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + LastTransitionTime: metav1.Now(), + }, + { + Type: "Ready", + Status: metav1.ConditionTrue, + Reason: "Ready", + LastTransitionTime: metav1.Now(), + }, + }, + SupportedKinds: []gatewayv1beta1.RouteGroupKind{ + { + Group: lo.ToPtr(gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group)), + Kind: "HTTPRoute", + }, + }, + }, + }, + } + require.NoError(t, client.Status().Patch(ctx, &gw, ctrlclient.MergeFrom(gwOld))) + + route := gatewayv1beta1.HTTPRoute{ + TypeMeta: metav1.TypeMeta{ + Kind: "HTTPRoute", + APIVersion: "v1beta1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: nsRoute.Name, + Name: uuid.NewString(), + }, + Spec: gatewayv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gatewayv1beta1.CommonRouteSpec{ + ParentRefs: []gatewayv1beta1.ParentReference{{ + Name: gatewayv1beta1.ObjectName(gw.Name), + Namespace: lo.ToPtr(gatewayv1beta1.Namespace(ns.Name)), + }}, + }, + Rules: []gatewayv1beta1.HTTPRouteRule{{ + BackendRefs: builder.NewHTTPBackendRef("backend-1").WithNamespace(ns.Name).ToSlice(), + }}, + }, + } + require.NoError(t, client.Create(ctx, &route)) + + nn := types.NamespacedName{ + Namespace: route.GetNamespace(), + Name: route.GetName(), + } + + t.Logf("verifying that HTTPRoute has ResolvedRefs set to Status False and Reason RefNotPermitted") + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "False", + Reason: "RefNotPermitted", + }, + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } + + rg := gatewayv1beta1.ReferenceGrant{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: ns.Name, + Name: uuid.NewString(), + }, + Spec: gatewayv1beta1.ReferenceGrantSpec{ + From: []gatewayv1beta1.ReferenceGrantFrom{ + { + Group: gatewayv1beta1.Group(gatewayv1beta1.GroupVersion.Group), + Kind: "HTTPRoute", + Namespace: gatewayv1beta1.Namespace(nsRoute.Name), + }, + }, + To: []gatewayv1beta1.ReferenceGrantTo{ + { + Group: "", + Kind: "Service", + }, + }, + }, + } + require.NoError(t, client.Create(ctx, &rg)) + t.Logf("verifying that HTTPRoute gets accepted by HTTPRouteReconciler after relevant ReferenceGrant gets created") + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "True", + Reason: "ResolvedRefs", + }, + metav1.Condition{ + Type: "Accepted", + Status: "True", + Reason: "Accepted", + }, + // Programmed condition requires a bit more work with mocks. + // It's set only when KubernetesObjectReports are enabled in the underlying + // dataplane client and then it relies on what's returned by + // dataplane client in KubernetesObjectConfigurationStatus(). + // This can be done but it's not the main focus of this test. + // Related: https://github.com/Kong/kubernetes-ingress-controller/issues/3793 + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } + + require.NoError(t, client.Delete(ctx, &rg)) + t.Logf("verifying that HTTPRoute gets its ResolvedRefs condition to Status False and Reason RefNotPermitted when relevant ReferenceGrant gets deleted") + + if !assert.Eventually(t, + helpers.HTTPRouteEventuallyContainsConditions(ctx, t, client, nn, + metav1.Condition{ + Type: "ResolvedRefs", + Status: "False", + Reason: "RefNotPermitted", + }, + ), + waitDuration, tickDuration, + ) { + t.Fatal(printHTTPRoutesConditions(ctx, client, nn)) + } + }) + } +} + +func printHTTPRoutesConditions(ctx context.Context, client ctrlclient.Client, nn types.NamespacedName) string { + var route gateway.HTTPRoute + err := client.Get(ctx, ctrlclient.ObjectKey{Namespace: nn.Namespace, Name: nn.Name}, &route) + if err != nil { + return fmt.Sprintf("failed to get HTTPRoute %s/%s when trying to print its conditions", nn.Namespace, nn.Name) + } + + if len(route.Status.Parents) == 0 { + return fmt.Sprintf("HTTPRoute %s/%s has no parents in Status", nn.Namespace, nn.Name) + } + + var sb strings.Builder + sb.WriteString(fmt.Sprintf("HTTPRoute %s/%s has the following Parents in Status:", nn.Namespace, nn.Name)) + for _, p := range route.Status.Parents { + if p.ParentRef.Namespace != nil { + _, _ = sb.WriteString(fmt.Sprintf("\nParent %s/%s: ", *p.ParentRef.Namespace, string(p.ParentRef.Name))) + } else { + _, _ = sb.WriteString(fmt.Sprintf("\nParent %s: ", string(p.ParentRef.Name))) + } + for _, c := range p.Conditions { + s := fmt.Sprintf( + "\n\tcondition: Type:%s, Status:%s, Reason:%s, ObservedGeneration:%d", + c.Type, c.Status, c.Reason, c.ObservedGeneration, + ) + _, _ = sb.WriteString(s) + } + _ = sb.WriteByte('\n') + } + return sb.String() +} diff --git a/internal/controllers/gateway/route_utils.go b/internal/controllers/gateway/route_utils.go index 4c117e39b7..746ce009d5 100644 --- a/internal/controllers/gateway/route_utils.go +++ b/internal/controllers/gateway/route_utils.go @@ -12,6 +12,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/controller-runtime/pkg/client" gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -487,7 +488,20 @@ func existsMatchingReadyListenerInStatus[T types.RouteT](route T, listener Liste // Find if the route has a type that's within the supported types, listed // in listener's status. _, ok := lo.Find(ls.SupportedKinds, func(rgk gatewayv1beta1.RouteGroupKind) bool { - gvk := route.GetObjectKind().GroupVersionKind() + // The artificially filled in GVK is needed for testing mostly and for + // situations when the object is not coming from the api server. + // Related upstream issue: https://github.com/kubernetes/kubernetes/issues/3030 + var gvk schema.GroupVersionKind + switch any(route).(type) { + case *HTTPRoute: + gvk = schema.GroupVersionKind{ + Group: gatewayv1beta1.GroupVersion.Group, + Version: gatewayv1beta1.GroupVersion.Version, + Kind: "HTTPRoute", + } + default: + gvk = route.GetObjectKind().GroupVersionKind() + } return (rgk.Group != nil && string(*rgk.Group) == gvk.Group) && string(rgk.Kind) == gvk.Kind }) return ok diff --git a/internal/util/builder/httpbackendref.go b/internal/util/builder/httpbackendref.go index eb5208c5f7..31620c5592 100644 --- a/internal/util/builder/httpbackendref.go +++ b/internal/util/builder/httpbackendref.go @@ -31,6 +31,10 @@ func (b *HTTPBackendRefBuilder) Build() gatewayv1beta1.HTTPBackendRef { return b.httpBackendRef } +func (b *HTTPBackendRefBuilder) ToSlice() []gatewayv1beta1.HTTPBackendRef { + return []gatewayv1beta1.HTTPBackendRef{b.httpBackendRef} +} + func (b *HTTPBackendRefBuilder) WithPort(port int) *HTTPBackendRefBuilder { val := gatewayv1beta1.PortNumber(port) b.httpBackendRef.Port = &val diff --git a/test/envtest/controller.go b/test/envtest/controller.go new file mode 100644 index 0000000000..9abefd8fb4 --- /dev/null +++ b/test/envtest/controller.go @@ -0,0 +1,53 @@ +package envtest + +import ( + "context" + "sync" + "testing" + + "github.com/bombsimon/logrusr/v2" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers" +) + +// StartReconciler creates a controller manager and starts the provided reconciler +// as its runnable. +// It also adds a t.Cleanup which waits for the maanger to exit so that the test +// can be self contained and logs from different tests' managers don't mix up. +func StartReconciler(ctx context.Context, t *testing.T, scheme *runtime.Scheme, cfg *rest.Config, reconciler controllers.Reconciler, opts *v1alpha1.ControllerManagerConfiguration) { + o := manager.Options{ + Logger: logrusr.New(logrus.New()), + Scheme: scheme, + MetricsBindAddress: "0", + } + if opts != nil { + var err error + o, err = o.AndFrom(opts) + require.NoError(t, err) + } + + mgr, err := ctrl.NewManager(cfg, o) + require.NoError(t, err) + + reconciler.SetLogger(mgr.GetLogger()) + + require.NoError(t, reconciler.SetupWithManager(mgr)) + + // This wait group makes it so that we wait for manager to exit. + // This way we get clean test logs not mixing between tests. + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + assert.NoError(t, mgr.Start(ctx)) + }() + t.Cleanup(func() { wg.Wait() }) +} diff --git a/test/envtest/k8s.go b/test/envtest/k8s.go index 2a11a373c2..e4ae80c1c6 100644 --- a/test/envtest/k8s.go +++ b/test/envtest/k8s.go @@ -13,6 +13,8 @@ import ( // CreateNamespace creates namespace using the provided client and returns it. func CreateNamespace(ctx context.Context, t *testing.T, client ctrlclient.Client) corev1.Namespace { + t.Helper() + ns := corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: uuid.NewString(), @@ -29,6 +31,8 @@ func CreateNamespace(ctx context.Context, t *testing.T, client ctrlclient.Client // CreatePod creates pod using the provided client and returns it. func CreatePod(ctx context.Context, t *testing.T, client ctrlclient.Client, ns string) corev1.Pod { + t.Helper() + pod := corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Namespace: ns, diff --git a/test/envtest/setup.go b/test/envtest/setup.go index 24c5fc6b3e..d1ae55347a 100644 --- a/test/envtest/setup.go +++ b/test/envtest/setup.go @@ -1,13 +1,23 @@ package envtest import ( + "go/build" + "os" + "os/signal" + "path/filepath" + "sync" + "syscall" "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/envtest" + gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/kong/kubernetes-ingress-controller/v2/test/consts" ) // Setup sets up the envtest environment which will be stopped on test cleanup @@ -15,21 +25,91 @@ import ( // // Note: If you want apiserver output on stdout set // KUBEBUILDER_ATTACH_CONTROL_PLANE_OUTPUT to true when running tests. -func Setup(t *testing.T) *rest.Config { +func Setup(t *testing.T, scheme *runtime.Scheme) *rest.Config { t.Helper() + gatewayCRDPath := filepath.Join(build.Default.GOPATH, "pkg", "mod", "sigs.k8s.io", "gateway-api@"+consts.GatewayAPIVersion, "config", "crd", "experimental") testEnv := &envtest.Environment{ ControlPlaneStopTimeout: time.Second * 60, + CRDDirectoryPaths: []string{ + gatewayCRDPath, + }, CRDInstallOptions: envtest.CRDInstallOptions{ CleanUpAfterUse: false, + Scheme: scheme, }, + Scheme: scheme, } + t.Logf("starting envtest environment...") cfg, err := testEnv.Start() require.NoError(t, err) + + t.Logf("waiting for Gateway API CRDs to be available...") + require.NoError(t, envtest.WaitForCRDs(cfg, []*v1.CustomResourceDefinition{ + { + Spec: v1.CustomResourceDefinitionSpec{ + Group: gatewayv1beta1.GroupVersion.Group, + Versions: []v1.CustomResourceDefinitionVersion{ + { + Name: gatewayv1beta1.GroupVersion.Version, + Served: true, + }, + }, + Names: v1.CustomResourceDefinitionNames{ + Plural: "gateways", + }, + }, + }, + { + Spec: v1.CustomResourceDefinitionSpec{ + Group: gatewayv1beta1.GroupVersion.Group, + Versions: []v1.CustomResourceDefinitionVersion{ + { + Name: gatewayv1beta1.GroupVersion.Version, + Served: true, + }, + }, + Names: v1.CustomResourceDefinitionNames{ + Plural: "httproutes", + }, + }, + }, + { + Spec: v1.CustomResourceDefinitionSpec{ + Group: gatewayv1beta1.GroupVersion.Group, + Versions: []v1.CustomResourceDefinitionVersion{ + { + Name: gatewayv1beta1.GroupVersion.Version, + Served: true, + }, + }, + Names: v1.CustomResourceDefinitionNames{ + Plural: "referencegrants", + }, + }, + }, + }, envtest.CRDInstallOptions{})) + + wg := sync.WaitGroup{} + wg.Add(1) + done := make(chan struct{}) + ch := make(chan os.Signal, 1) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM) + go func() { + defer wg.Done() + select { + case <-ch: + _ = testEnv.Stop() + case <-done: + _ = testEnv.Stop() + } + }() + t.Cleanup(func() { t.Logf("stopping envtest environment for test %s", t.Name()) - assert.NoError(t, testEnv.Stop()) + close(done) + wg.Wait() }) return cfg diff --git a/test/helpers/conditions.go b/test/helpers/conditions.go new file mode 100644 index 0000000000..a97dbe82bb --- /dev/null +++ b/test/helpers/conditions.go @@ -0,0 +1,66 @@ +package helpers + +import ( + "context" + "testing" + + "github.com/samber/lo" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/net" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kong/kubernetes-ingress-controller/v2/internal/controllers/gateway" +) + +// TODO: for now this can stay here but ideally we'd use a common package for this +// and github.com/kong/kubernetes-ingress-controller/v2/test/interla/helpers. +// At the moment we can't use the test/internal package in e.g. internal/controllers +// package because of how the internal packages work. +// This might require a separate PR that will reorder code and put it in a top +// level internal package instead of test/internal. + +// HTTPRouteEventuallyContainsConditions returns a predicate function that can be +// used with assert.Eventually or require.Eventually in order to check - via the +// provided client - that the HTTPRoute with the NamespacedName as provided in +// the arguments, does indeed contain the provied conditions in the status. +func HTTPRouteEventuallyContainsConditions(ctx context.Context, t *testing.T, client ctrlclient.Client, nn types.NamespacedName, conds ...metav1.Condition) func() bool { + return func() bool { + t.Helper() + + var ( + ns = nn.Namespace + name = nn.Name + route = gateway.HTTPRoute{} + ) + + err := client.Get(ctx, ctrlclient.ObjectKey{Namespace: ns, Name: name}, &route) + if err != nil { + // No point in continuing if connection is down. + if net.IsConnectionRefused(err) { + require.NoError(t, err) + return false + } + t.Logf("failed to get HTTPRoute: %v", err) + return false + } + + return lo.ContainsBy(route.Status.Parents, func(p gateway.RouteParentStatus) bool { + var count int + for _, cond := range conds { + contains := lo.ContainsBy(p.Conditions, func(c metav1.Condition) bool { + return c.Type == cond.Type && c.Status == cond.Status && c.Reason == cond.Reason + }) + if !contains { + t.Logf("condition Type:%s, Status:%s, Reason:%s missing from route:%s/%s", + cond.Type, cond.Status, cond.Reason, ns, name, + ) + return false + } + count++ + } + return count == len(conds) + }) + } +}