Proposal: add e2e tests based on k8s e2e-framework

.github/workflows/ci.yml

@@ -231,6 +231,67 @@ jobs:
       - name: Run E2E Tests
         run: make e2e USE_HELM3=true
 
+  e2e-alternative-tests:
+    runs-on: ubuntu-20.04
+    needs: detect-noop
+    if: needs.detect-noop.outputs.noop != 'true'
+
+    steps:
+      - name: Setup QEMU
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: all
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: ${{ env.DOCKER_BUILDX_VERSION }}
+          install: true
+
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          submodules: true
+
+      - name: Fetch History
+        run: git fetch --prune --unshallow
+
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+          go-version: ${{ env.GO_VERSION }}
+
+      - name: Find the Go Build Cache
+        id: go
+        run: echo "::set-output name=cache::$(make go.cachedir)"
+
+      - name: Cache the Go Build Cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ steps.go.outputs.cache }}
+          key: ${{ runner.os }}-build-e2e-tests-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-build-e2e-tests-
+
+      - name: Cache Go Dependencies
+        uses: actions/cache@v2
+        with:
+          path: .work/pkg
+          key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-pkg-
+
+      - name: Vendor Dependencies
+        run: make vendor vendor.check
+
+      - name: Run E2E Tests
+        run: make e2e.alternative-run
+
+      - name: Publish Logs to GitHub
+        uses: actions/upload-artifact@v2
+        with:
+          name: e2e-logs
+          path: e2e/logs/**
+
   publish-artifacts:
     runs-on: ubuntu-20.04
     needs: detect-noop

.gitignore

@@ -13,3 +13,5 @@ cover.out
 .idea/
 # Ignore kubeconfig generated by `make dev-debug`
 kubeconfig
+
+e2e/logs

.golangci.yml

@@ -1,7 +1,8 @@
 run:
   deadline: 20m
   timeout: 20m
-
+  build-tags:
+    - e2e
   skip-files:
   - "zz_generated\\..+\\.go$"
 

Makefile

@@ -66,6 +66,14 @@ XPKGS = provider-argocd
 # we ensure image is present in daemon.
 xpkg.build.provider-argocd: do.build.images
 
+# Setup E2E tests based on e2e-framework
+E2E_FILTER ?= .*
+E2E_CONTROLLER ?= $(BUILD_REGISTRY)/$(subst crossplane-,crossplane/,$(PROJECT_NAME)):latest
+E2E_XPKG_TAR ?= $(XPKG_OUTPUT_DIR)/$(PLATFORM)/$(PROJECT_NAME)-$(VERSION).xpkg
+E2E_XPKG_TAG ?= e2e/$(PROJECT_NAME):$(VERSION)
+
+export E2E_IMAGES = {"crossplane/provider-argocd":"${E2E_XPKG_TAG}"}
+
 # ====================================================================================
 # Targets
 
@@ -150,6 +158,20 @@ dev-teardown: $(KIND) $(KUBECTL)
 
 .PHONY: cobertura manifests submodules fallthrough test-integration run crds.clean
 
+
+.PHOBY: e2e.alternative-run
+
+e2e.alternative-run: $(KIND) $(HELM3) build e2e.alternative-run.load-image
+	@$(INFO) running e2e tests
+	@echo "E2E_IMAGES=$$E2E_IMAGES"
+	go test -v  $(PROJECT_REPO)/e2e/... -tags=e2e -count=1 -test.v -run '$(E2E_FILTER)'
+	@$(OK) e2e tests passed
+
+e2e.alternative-run.load-image:
+
+	$(eval DIGEST=$(shell docker image load -i $(E2E_XPKG_TAR) | cut -d ":" -f3))
+
+	docker tag $(DIGEST) $(E2E_XPKG_TAG)
 # ====================================================================================
 # Special Targets
 

e2e/application_v1alpha1_test.go

@@ -0,0 +1,27 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/crossplane-contrib/xp-testing/pkg/resources"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	"github.com/crossplane-contrib/provider-argocd/apis/applications/v1alpha1"
+)
+
+func Test_Application_v1alpha1(t *testing.T) {
+
+	resource := resources.NewResourceTestConfig(&v1alpha1.Application{}, "Application")
+
+	fB := features.New(fmt.Sprintf("%v", resource.Kind))
+	fB.WithLabel("kind", resource.Kind)
+	fB.Setup(resource.Setup)
+	fB.Assess("create", resource.AssessCreate)
+	fB.Assess("delete", resource.AssessDelete)
+
+	testenv.Test(t, fB.Feature())
+
+}

e2e/cluster_v1alpha1_test.go

@@ -0,0 +1,33 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"path"
+	"testing"
+
+	"github.com/crossplane-contrib/xp-testing/pkg/resources"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	"github.com/crossplane-contrib/provider-argocd/apis/cluster/v1alpha1"
+)
+
+// Example how to use subdirectories
+func Test_Cluster_v1alpha1_simple(t *testing.T) {
+
+	resource := resources.ResourceTestConfig{
+		Kind:              "Cluster",
+		Obj:               &v1alpha1.Cluster{},
+		ResourceDirectory: path.Join(resources.DefaultCRFolder("Cluster"), "simple"),
+	}
+
+	fB := features.New(fmt.Sprintf("%v-%v", resource.Kind, "simple"))
+	fB.WithLabel("kind", resource.Kind)
+	fB.Setup(resource.Setup)
+	fB.Assess("create", resource.AssessCreate)
+	fB.Assess("delete", resource.AssessDelete)
+
+	testenv.Test(t, fB.Feature())
+
+}

e2e/crs/Application/cluster.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cluster.argocd.crossplane.io/v1alpha1
+kind: Cluster
+metadata:
+  name: example-cluster-kubeconfig
+spec:
+  forProvider:
+    server: https://kubernetes.default.svc
+    name: example-cluster-kubeconfig
+    config:
+      tlsClientConfig:
+        insecure: true
+  providerConfigRef:
+    name: argocd-provider

e2e/crs/Cluster/simple/cluster.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cluster.argocd.crossplane.io/v1alpha1
+kind: Cluster
+metadata:
+  name: example-cluster
+spec:
+  forProvider:
+    server: https://kubernetes.default.svc
+    name: example-cluster
+    config:
+      tlsClientConfig:
+        insecure: true
+  providerConfigRef:
+    name: argocd-provider

e2e/crs/Project/project.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: projects.argocd.crossplane.io/v1alpha1
+kind: Project
+metadata:
+  name: example-project
+spec:
+  forProvider:
+    description: "example project for e2e tests"
+  providerConfigRef:
+    name: argocd-provider

e2e/crs/Repository/repository.yaml

@@ -0,0 +1,11 @@
+---
+apiVersion: repositories.argocd.crossplane.io/v1alpha1
+kind: Repository
+metadata:
+  name: example-project.git
+spec:
+  forProvider:
+    repo: https://github.com/stefanprodan/podinfo.git
+    type: git
+  providerConfigRef:
+    name: argocd-provider

e2e/main_test.go

@@ -0,0 +1,50 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"os"
+	"testing"
+
+	runtime "k8s.io/apimachinery/pkg/runtime"
+
+	"github.com/crossplane-contrib/provider-argocd/apis"
+
+	xpv1alpha1 "github.com/crossplane/crossplane/apis/pkg/v1alpha1"
+	"sigs.k8s.io/e2e-framework/pkg/env"
+
+	"github.com/crossplane-contrib/xp-testing/pkg/images"
+	"github.com/crossplane-contrib/xp-testing/pkg/logging"
+	"github.com/crossplane-contrib/xp-testing/pkg/setup"
+)
+
+var testenv env.Environment
+
+func TestMain(m *testing.M) {
+	var verbosity = 4
+	logging.EnableVerboseLogging(&verbosity)
+	testenv = env.NewParallel()
+
+	key := "crossplane/provider-argocd"
+	imgs := images.GetImagesFromEnvironmentOrPanic(key, &key)
+	// Enhance interface for one- based providers
+	clusterSetup := setup.ClusterSetup{
+		Name:   "argocd",
+		Images: imgs,
+		ControllerConfig: &xpv1alpha1.ControllerConfig{
+			Spec: xpv1alpha1.ControllerConfigSpec{
+				Image: &imgs.Package,
+				// Raise sync interval to speed up tests
+				// add debug output, in case necessary for debugging in e.g. CI
+				Args: []string{"--debug", "--sync=5s"},
+			},
+		},
+		SecretData:        nil,
+		AddToSchemaFuncs:  []func(s *runtime.Scheme) error{apis.AddToScheme},
+		CrossplaneVersion: "1.13.2",
+	}
+
+	clusterSetup.Configure(testenv)
+	testenv.Setup(installArgocd("v2.7.4"))
+	os.Exit(testenv.Run(m))
+}

e2e/project_v1alpha1_test.go

@@ -0,0 +1,26 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"testing"
+
+	"github.com/crossplane-contrib/xp-testing/pkg/resources"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	"github.com/crossplane-contrib/provider-argocd/apis/projects/v1alpha1"
+)
+
+func Test_Project_v1alpha1(t *testing.T) {
+
+	resource := resources.NewResourceTestConfig(&v1alpha1.Project{}, "Project")
+
+	fB := features.New(resource.Kind)
+	fB.WithLabel("kind", resource.Kind)
+	fB.Setup(resource.Setup)
+	fB.Assess("create", resource.AssessCreate)
+	fB.Assess("delete", resource.AssessDelete)
+
+	testenv.Test(t, fB.Feature())
+
+}

e2e/provider/local.yaml

@@ -0,0 +1,16 @@
+---
+# argocd provider that references the secret credentials
+apiVersion: argocd.crossplane.io/v1alpha1
+kind: ProviderConfig
+metadata:
+  name: argocd-provider
+spec:
+  serverAddr: argocd-server.argocd.svc:443
+  insecure: true
+  plainText: false
+  credentials:
+    source: Secret
+    secretRef:
+      namespace: crossplane-system
+      name: argocd-credentials
+      key: authToken

e2e/repository_v1alpha1_test.go

@@ -0,0 +1,32 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"path"
+	"testing"
+
+	"github.com/crossplane-contrib/xp-testing/pkg/resources"
+	"sigs.k8s.io/e2e-framework/pkg/features"
+
+	"github.com/crossplane-contrib/provider-argocd/apis/repositories/v1alpha1"
+)
+
+func Test_Repositories_v1alpha1(t *testing.T) {
+
+	resource := resources.ResourceTestConfig{
+		Kind:              "Repository",
+		Obj:               &v1alpha1.Repository{},
+		ResourceDirectory: path.Join(resources.DefaultCRFolder("Repository")),
+	}
+
+	fB := features.New(fmt.Sprintf("%v", resource.Kind))
+	fB.WithLabel("kind", resource.Kind)
+	fB.Setup(resource.Setup)
+	fB.Assess("create", resource.AssessCreate)
+	fB.Assess("delete", resource.AssessDelete)
+
+	testenv.Test(t, fB.Feature())
+
+}