test: Implemented e2e test framework

Makefile

@@ -64,7 +64,7 @@ vet: ## Vet the code and dependencies
 
 .PHONY: test
 test: ## Run tests.
-	go test ./... -coverprofile coverage.out
+	go test ./pkg/... -coverprofile coverage.out
 
 .PHONY: toolchain
 toolchain: ## Install developer toolchain
@@ -86,3 +86,8 @@ docker-push: ## Push docker image with the manager.
 build-deploy: ## Create a deployment file that can be applied with `kubectl apply -f deploy.yaml`
 	cd config/manager && kustomize edit set image controller=${ECRIMAGES}
 	kustomize build config/default > deploy.yaml
+
+## Run e2e tests against cluster pointed to by ~/.kube/config
+.PHONY: e2etest
+e2etest:
+	cd test && go test -v ./... -count=1 --ginkgo.v

test/go.mod

@@ -0,0 +1,88 @@
+module github.com/aws/aws-application-networking-k8s/test
+
+go 1.20
+
+replace (
+	github.com/aws/aws-application-networking-k8s => ../
+	github.com/aws/aws-sdk-go => ../scripts/aws_sdk_model_override/aws-sdk-go
+)
+
+require (
+	github.com/Pallinder/go-randomdata v1.2.0
+	github.com/aws/aws-application-networking-k8s v0.3.0
+	github.com/aws/aws-sdk-go v1.42.18
+	github.com/imdario/mergo v0.3.13
+	github.com/onsi/ginkgo/v2 v2.9.1
+	github.com/onsi/gomega v1.27.3
+	github.com/samber/lo v1.37.0
+	go.uber.org/zap v1.24.0
+	k8s.io/api v0.26.2
+	k8s.io/apimachinery v0.26.2
+	k8s.io/client-go v0.26.2
+	sigs.k8s.io/controller-runtime v0.14.5
+	sigs.k8s.io/gateway-api v0.6.1
+	sigs.k8s.io/mcs-api v0.1.0
+)
+
+require (
+	github.com/benbjohnson/clock v1.1.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.2 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/emicklei/go-restful/v3 v3.9.0 // indirect
+	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
+	github.com/fsnotify/fsnotify v1.6.0 // indirect
+	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-openapi/jsonpointer v0.19.5 // indirect
+	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/swag v0.19.14 // indirect
+	github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/glog v1.0.0 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/mock v1.6.0 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/gnostic v0.5.7-v3refs // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
+	github.com/google/uuid v1.1.2 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/procfs v0.8.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	go.uber.org/atomic v1.7.0 // indirect
+	go.uber.org/multierr v1.6.0 // indirect
+	golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect
+	golang.org/x/net v0.8.0 // indirect
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+	golang.org/x/tools v0.7.0 // indirect
+	gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	k8s.io/apiextensions-apiserver v0.26.1 // indirect
+	k8s.io/component-base v0.26.1 // indirect
+	k8s.io/klog/v2 v2.80.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448 // indirect
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
+)

test/pkg/test/context.go

@@ -0,0 +1,24 @@
+package test
+
+import (
+	"context"
+	"testing"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest"
+)
+
+type loggerKey struct{}
+
+func NewContext(t *testing.T) context.Context {
+	ctx := context.Background()
+	ctx = context.WithValue(ctx, loggerKey{}, zaptest.NewLogger(t, zaptest.WrapOptions(
+		zap.AddCaller(),
+		zap.Development(),
+	)).Sugar())
+	return ctx
+}
+
+func Logger(ctx context.Context) *zap.SugaredLogger {
+	return ctx.Value(loggerKey{}).(*zap.SugaredLogger)
+}

test/pkg/test/framework.go

@@ -0,0 +1,254 @@
+package test
+
+import (
+	"context"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/aws/aws-application-networking-k8s/pkg/aws/services"
+	"github.com/aws/aws-application-networking-k8s/pkg/latticestore"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/vpclattice"
+	. "github.com/onsi/ginkgo/v2"
+	"github.com/onsi/gomega"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/format"
+	"github.com/onsi/gomega/types"
+	"github.com/samber/lo"
+	appsv1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/client-go/kubernetes/scheme"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/gateway-api/apis/v1beta1"
+	"sigs.k8s.io/mcs-api/pkg/apis/v1alpha1"
+)
+
+func init() {
+	format.MaxLength = 0
+}
+
+var (
+	CleanupTimeout  = 300 * time.Second
+	CreationTimeout = 120 * time.Second
+	TestObjects     = []struct {
+		Type     client.Object
+		ListType client.ObjectList
+	}{
+		// Must currently be deleted in order to avoid https://github.com/aws/aws-application-networking-k8s/issues/115
+		{&v1.Service{}, &v1.ServiceList{}},
+		{&appsv1.Deployment{}, &appsv1.DeploymentList{}},
+		{&v1beta1.HTTPRoute{}, &v1beta1.HTTPRouteList{}},
+		{&v1beta1.Gateway{}, &v1beta1.GatewayList{}},
+	}
+)
+
+type Framework struct {
+	client.Client
+	LatticeClient services.Lattice
+}
+
+func NewFramework(ctx context.Context) *Framework {
+	var scheme = scheme.Scheme
+	lo.Must0(v1beta1.Install(scheme))
+	lo.Must0(v1alpha1.Install(scheme))
+	framework := &Framework{
+		Client:        lo.Must(client.New(controllerruntime.GetConfigOrDie(), client.Options{Scheme: scheme})),
+		LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), ""), // region is currently hardcoded
+	}
+	gomega.Default.SetDefaultEventuallyPollingInterval(time.Second * 1)
+	BeforeEach(func() { framework.ExpectToBeClean(ctx) })
+	AfterSuite(func() { framework.ExpectToClean(ctx) })
+	return framework
+}
+
+func (env *Framework) ExpectToBeClean(ctx context.Context) {
+	Logger(ctx).Info("Expecting the test environment to be clean")
+	// Kubernetes API Objects
+	for _, testObject := range TestObjects {
+		env.EventuallyExpectNoneFound(ctx, testObject.ListType).WithOffset(1).Should(Succeed())
+	}
+	// AWS API Objects
+	Eventually(func(g Gomega) {
+		g.Expect(env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{})).To(HaveField("Items", BeEmpty()))
+		g.Expect(env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{})).To(HaveField("Items", BeEmpty()))
+		g.Expect(env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{})).To(HaveField("Items", BeEmpty()))
+	})
+}
+
+func (env *Framework) ExpectToClean(ctx context.Context) {
+	Logger(ctx).Info("Cleaning the test environment")
+	wg := sync.WaitGroup{}
+	namespaces := &v1.NamespaceList{}
+	// Kubernetes API Objects
+	Expect(env.List(ctx, namespaces)).WithOffset(1).To(Succeed())
+	for _, namespace := range namespaces.Items {
+		for _, object := range TestObjects {
+			wg.Add(1)
+			go func(object client.Object, objectList client.ObjectList, namespace string) {
+				defer wg.Done()
+				defer GinkgoRecover()
+				env.ExpectDeleteAllToSucceed(ctx, object, namespace)
+				env.EventuallyExpectNoneFound(ctx, objectList).Should(Succeed())
+			}(object.Type.DeepCopyObject().(client.Object), object.ListType.DeepCopyObject().(client.ObjectList), namespace.Name)
+		}
+	}
+	wg.Wait()
+
+	// AWS API Objects
+	// Delete Services
+	listServicesOutput, err := env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{})
+	Expect(err).ToNot(HaveOccurred())
+	for _, service := range listServicesOutput.Items {
+		// Delete ServiceNetworkServiceAssociations
+		listServiceNetworkServiceAssociationsOutput, err := env.LatticeClient.ListServiceNetworkServiceAssociationsWithContext(ctx, &vpclattice.ListServiceNetworkServiceAssociationsInput{ServiceIdentifier: service.Id})
+		Expect(err).ToNot(HaveOccurred())
+		for _, serviceNetworkServiceAssociation := range listServiceNetworkServiceAssociationsOutput.Items {
+			_, err := env.LatticeClient.DeleteServiceNetworkServiceAssociationWithContext(ctx, &vpclattice.DeleteServiceNetworkServiceAssociationInput{ServiceNetworkServiceAssociationIdentifier: serviceNetworkServiceAssociation.Id})
+			Expect(err).ToNot(HaveOccurred())
+		}
+		// Delete Listeners
+		listListenersOutput, err := env.LatticeClient.ListListenersWithContext(ctx, &vpclattice.ListListenersInput{ServiceIdentifier: service.Id})
+		Expect(err).ToNot(HaveOccurred())
+		for _, listener := range listListenersOutput.Items {
+			_, err = env.LatticeClient.DeleteListenerWithContext(ctx, &vpclattice.DeleteListenerInput{ServiceIdentifier: service.Id, ListenerIdentifier: listener.Id})
+			Expect(err).ToNot(HaveOccurred())
+		}
+		// Delete Service
+		_, err = env.LatticeClient.DeleteServiceWithContext(ctx, &vpclattice.DeleteServiceInput{ServiceIdentifier: service.Id})
+		Expect(err).ToNot(HaveOccurred())
+	}
+	// Delete TargetGroups
+	listTargetGroupsOutput, err := env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{})
+	Expect(err).ToNot(HaveOccurred())
+	for _, targetGroup := range listTargetGroupsOutput.Items {
+		// Delete Targets
+		listTargetsOutput, err := env.LatticeClient.ListTargetsWithContext(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: targetGroup.Id})
+		Expect(err).ToNot(HaveOccurred())
+		if targets := lo.Map(listTargetsOutput.Items, func(target *vpclattice.TargetSummary, _ int) *vpclattice.Target {
+			return &vpclattice.Target{Id: target.Id}
+		}); len(targets) > 0 {
+			_, err = env.LatticeClient.DeregisterTargetsWithContext(ctx, &vpclattice.DeregisterTargetsInput{TargetGroupIdentifier: targetGroup.Id, Targets: targets})
+			Expect(err).ToNot(HaveOccurred())
+		}
+		// Delete TargetGroup
+		_, err = env.LatticeClient.DeleteTargetGroupWithContext(ctx, &vpclattice.DeleteTargetGroupInput{TargetGroupIdentifier: targetGroup.Id})
+		Expect(err).ToNot(HaveOccurred())
+	}
+	listServiceNetworksOutput, err := env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{})
+	Expect(err).ToNot(HaveOccurred())
+	for _, serviceNetwork := range listServiceNetworksOutput.Items {
+		_, err := env.LatticeClient.DeleteServiceNetworkWithContext(ctx, &vpclattice.DeleteServiceNetworkInput{ServiceNetworkIdentifier: serviceNetwork.Id})
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	// Wait for objects to delete
+	env.ExpectToBeClean(ctx)
+}
+
+func (env *Framework) ExpectCreated(ctx context.Context, objects ...client.Object) {
+	for _, object := range objects {
+		Logger(ctx).Infof("Creating %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName())
+		Expect(env.Create(ctx, object)).WithOffset(1).To(Succeed())
+	}
+}
+
+func (env *Framework) ExpectDeleted(ctx context.Context, objects ...client.Object) {
+	for _, object := range objects {
+		Logger(ctx).Infof("Deleting %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName())
+		Expect(env.Delete(ctx, object)).WithOffset(1).To(Succeed())
+	}
+}
+
+func (env *Framework) ExpectDeleteAllToSucceed(ctx context.Context, object client.Object, namespace string) {
+	Expect(env.DeleteAllOf(ctx, object, client.InNamespace(namespace), client.HasLabels([]string{DiscoveryLabel}))).WithOffset(1).To(Succeed())
+}
+
+func (env *Framework) EventuallyExpectNotFound(ctx context.Context, objects ...client.Object) types.AsyncAssertion {
+	return Eventually(func(g Gomega) {
+		for _, object := range objects {
+			g.Expect(errors.IsNotFound(env.Get(ctx, client.ObjectKeyFromObject(object), object))).To(BeTrue())
+		}
+	}, CleanupTimeout)
+}
+
+func (env *Framework) EventuallyExpectNoneFound(ctx context.Context, objectList client.ObjectList) types.AsyncAssertion {
+	return Eventually(func(g Gomega) {
+		g.Expect(env.List(ctx, objectList, client.HasLabels([]string{DiscoveryLabel}))).To(Succeed())
+		g.Expect(meta.ExtractList(objectList)).To(HaveLen(0), "Expected to not find any %q with label %q", reflect.TypeOf(objectList), DiscoveryLabel)
+	}, CleanupTimeout)
+}
+
+func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *v1beta1.Gateway) *vpclattice.ServiceNetworkSummary {
+	var found *vpclattice.ServiceNetworkSummary
+	Eventually(func(g Gomega) {
+		listServiceNetworksOutput, err := env.LatticeClient.ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(listServiceNetworksOutput.Items).ToNot(BeEmpty())
+		for _, serviceNetwork := range listServiceNetworksOutput.Items {
+			if lo.FromPtr(serviceNetwork.Name) == gateway.Name {
+				found = serviceNetwork
+			}
+		}
+		g.Expect(found).ToNot(BeNil())
+	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	return found
+}
+
+func (env *Framework) GetService(ctx context.Context, httpRoute *v1beta1.HTTPRoute) *vpclattice.ServiceSummary {
+	var found *vpclattice.ServiceSummary
+	Eventually(func(g Gomega) {
+		listServicesOutput, err := env.LatticeClient.ListServicesWithContext(ctx, &vpclattice.ListServicesInput{})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(listServicesOutput.Items).ToNot(BeEmpty())
+		for _, service := range listServicesOutput.Items {
+			if lo.FromPtr(service.Name) == latticestore.AWSServiceName(httpRoute.Name, httpRoute.Namespace) {
+				found = service
+			}
+		}
+		g.Expect(found).ToNot(BeNil())
+		g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.ServiceStatusActive)))
+	}, CreationTimeout).WithOffset(1).Should(Succeed())
+
+	return found
+}
+
+func (env *Framework) GetTargetGroup(ctx context.Context, service *v1.Service) *vpclattice.TargetGroupSummary {
+	var found *vpclattice.TargetGroupSummary
+	Eventually(func(g Gomega) {
+		listTargetGroupsOutput, err := env.LatticeClient.ListTargetGroupsWithContext(ctx, &vpclattice.ListTargetGroupsInput{})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(listTargetGroupsOutput.Items).ToNot(BeEmpty())
+		for _, targetGroup := range listTargetGroupsOutput.Items {
+			if lo.FromPtr(targetGroup.Name) == latticestore.TargetGroupName(service.Name, service.Namespace) {
+				found = targetGroup
+			}
+		}
+		g.Expect(found).ToNot(BeNil())
+		g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.TargetGroupStatusActive)))
+	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	return found
+}
+
+func (env *Framework) GetTargets(ctx context.Context, targetGroup *vpclattice.TargetGroupSummary, deployment *appsv1.Deployment) []*vpclattice.TargetSummary {
+	var found []*vpclattice.TargetSummary
+	Eventually(func(g Gomega) {
+		podList := &v1.PodList{}
+		g.Expect(env.List(ctx, podList, client.MatchingLabels(deployment.Spec.Selector.MatchLabels))).To(Succeed())
+		g.Expect(podList.Items).To(HaveLen(int(*deployment.Spec.Replicas)))
+
+		listTargetsOutput, err := env.LatticeClient.ListTargetsWithContext(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: targetGroup.Id})
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(listTargetsOutput.Items).To(HaveLen(int(*deployment.Spec.Replicas)))
+
+		podIps := lo.Map(podList.Items, func(pod v1.Pod, _ int) string { return pod.Status.PodIP })
+		targetIps := lo.Filter(listTargetsOutput.Items, func(target *vpclattice.TargetSummary, _ int) bool { return lo.Contains(podIps, *target.Id) })
+		g.Expect(targetIps).To(HaveLen(int(*deployment.Spec.Replicas)))
+
+		found = listTargetsOutput.Items
+	}, CreationTimeout).WithOffset(1).Should(Succeed())
+	return found
+}