E2E test improvements

- Run the controller in e2e test
- Combine wordpress test with the existing e2e test
- Use a kind cluster to do real e2e

.gitignore

@@ -1,3 +1,7 @@
+# e2e pid files
+manager.pid
+manager.log
+
 # OSX leaves these everywhere on SMB shares
 ._*
 

Makefile

@@ -90,7 +90,7 @@ test: $(TOOLBIN)/etcd $(TOOLBIN)/kube-apiserver $(TOOLBIN)/kubectl
 # Run e2e-tests
 .PHONY: e2e-test
 e2e-test: generate fmt vet manifests $(TOOLBIN)/kind
-	BIN=$(TOOLBIN) ./e2e/test_e2e.sh
+	BIN=$(TOOLBIN) ./e2e/run_e2e_test.sh
 
 ## --------------------------------------
 ## Build and run
@@ -100,10 +100,15 @@ e2e-test: generate fmt vet manifests $(TOOLBIN)/kind
 bin/manager: main.go generate fmt vet manifests
 	go build -o bin/manager main.go
 
+# Run against the configured Kubernetes cluster in ~/.kube/config
+.PHONY: runbg
+runbg: bin/manager
+	bin/manager --metrics-addr ":8083" >& manager.log & echo $$! > manager.pid
+
 # Run against the configured Kubernetes cluster in ~/.kube/config
 .PHONY: run
-run: generate fmt vet manifests
-	go run ./main.go
+run: bin/manager
+	bin/manager
 
 # Debug using the configured Kubernetes cluster in ~/.kube/config
 .PHONY: debug
@@ -153,36 +158,36 @@ misspell-fix: $(TOOLBIN)/misspell
 
 # Install CRDs into a cluster
 .PHONY: install
-install: $(TOOLBIN)/kustomize
-	$(TOOLBIN)/kustomize build config/crd| kubectl apply -f -
+install: $(TOOLBIN)/kustomize $(TOOLBIN)/kubectl
+	$(TOOLBIN)/kustomize build config/crd| $(TOOLBIN)/kubectl apply -f -
 
 # Uninstall CRDs from a cluster
 .PHONY: uninstall
-uninstall: $(TOOLBIN)/kustomize
-	$(TOOLBIN)/kustomize build config/crd| kubectl delete -f -
+uninstall: $(TOOLBIN)/kustomize $(TOOLBIN)/kubectl
+	$(TOOLBIN)/kustomize build config/crd| $(TOOLBIN)/kubectl delete -f -
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
 .PHONY: deploy
-deploy: $(TOOLBIN)/kustomize
+deploy: $(TOOLBIN)/kustomize $(TOOLBIN)/kubectl
 	cd config/manager && $(TOOLBIN)/kustomize edit set image controller=$(CONTROLLER_IMG)-$(ARCH):$(TAG)
-	$(TOOLBIN)/kustomize build config/default | kubectl apply -f -
+	$(TOOLBIN)/kustomize build config/default | $(TOOLBIN)/kubectl apply -f -
 
 # unDeploy controller in the configured Kubernetes cluster in ~/.kube/config
 .PHONY: undeploy
-undeploy: $(TOOLBIN)/kustomize
-	$(TOOLBIN)/kustomize build config/default | kubectl delete -f -
+undeploy: $(TOOLBIN)/kustomize $(TOOLBIN)/kubectl
+	$(TOOLBIN)/kustomize build config/default | $(TOOLBIN)/kubectl delete -f -
 
 # Deploy wordpress
 .PHONY: deploy-wordpress
-deploy-wordpress: $(TOOLBIN)/kustomize
+deploy-wordpress: $(TOOLBIN)/kustomize $(TOOLBIN)/kubectl
 	mkdir -p /tmp/data1 /tmp/data2
-	$(TOOLBIN)/kustomize build docs/examples/wordpress | kubectl apply -f -
+	$(TOOLBIN)/kustomize build docs/examples/wordpress | $(TOOLBIN)/kubectl apply -f -
 
 # Uneploy wordpress
 .PHONY: undeploy-wordpress
-undeploy-wordpress: $(TOOLBIN)/kustomize
-	$(TOOLBIN)/kustomize build docs/examples/wordpress | kubectl delete -f -
-	# kubectl delete pvc --all
+undeploy-wordpress: $(TOOLBIN)/kustomize $(TOOLBIN)/kubectl
+	$(TOOLBIN)/kustomize build docs/examples/wordpress | $(TOOLBIN)/kubectl delete -f -
+	# $(TOOLBIN)/kubectl delete pvc --all
 	# sudo rm -fr /tmp/data1 /tmp/data2
 
 ## --------------------------------------

README.md

@@ -219,10 +219,10 @@ spec:
         type: "image/png"
     type: "wordpress"
     maintainers:
-      - name: Kenneth Owens
-        email: kow3ns@github.com
+      - name: Wordpress Dev
+        email: wpdev@domain.com
     owners:
-      - "Kenneth Owens kow3ns@github.com"
+      - "Wordpress Admin wpadmin@domain.com"
     keywords:
       - "cms"
       - "blog"

config/crd/bases/app.k8s.io_applications.yaml

@@ -1,6 +1,3 @@
-# Copyright 2020 The Kubernetes Authors.
-# SPDX-License-Identifier: Apache-2.0
-
 
 ---
 apiVersion: apiextensions.k8s.io/v1beta1

config/rbac/role.yaml

@@ -1,6 +1,3 @@
-# Copyright 2020 The Kubernetes Authors.
-# SPDX-License-Identifier: Apache-2.0
-
 
 ---
 apiVersion: rbac.authorization.k8s.io/v1

config/samples/app_v1beta1_application.yaml

@@ -28,11 +28,11 @@ spec:
       type: "image/png"
     assemblyPhase: "Pending"
     maintainers:
-    - name: Kenneth Owens
-      email: kow3ns@github.com
+    - name: wordpress dev
+      email: wpdev@cwdomaingithub.com
     owners:
-    - name: Kenneth Owens
-      email: kow3ns@github.com
+    - name: wordpress admin
+      email: wpadmin@github.com
     keywords:
     - "cms"
     - "blog"

docs/examples/example.yaml

@@ -27,11 +27,11 @@ spec:
       type: "image/png"
       size: "2000x680"
   maintainers:
-    - name: Kenneth Owens
-      email: kow3ns@github.com
+    - name: App Admin
+      email: appadmin@domain.com
   owners:
-    - name: Kenneth Owens
-      email: kow3ns@github.com
+    - name: App Owner
+      email: appowner@domain.com
   keywords:
    - "cms"
    - "blog"

docs/examples/wordpress/application.yaml

@@ -27,11 +27,11 @@ spec:
       type: "image/png"
       size: "2000x680"
   maintainers:
-    - name: Kenneth Owens
-      email: kow3ns@github.com
+    - name: MySQL Administrator
+      email: mysqladmin@domain.com
   owners:
-    - name: Kenneth Owens
-      email: kow3ns@github.com
+    - name: MySQL Administrator
+      email: mysqladmin@domain.com
   addOwnerRef: true
   keywords:
    - "cms"

e2e/kind-config.yaml

@@ -6,3 +6,6 @@ kind: Cluster
 apiVersion: kind.sigs.k8s.io/v1alpha3
 nodes:
   - role: control-plane
+  - role: worker
+  - role: worker
+  - role: worker

e2e/main_test.go

@@ -4,16 +4,26 @@
 package main
 
 import (
+	"bytes"
+	"context"
+	"io"
 	"log"
 	"os"
+	"os/exec"
 	"path"
 	"testing"
+	"time"
 
 	. "github.com/onsi/ginkgo"
 	"github.com/onsi/ginkgo/reporters"
 	. "github.com/onsi/gomega"
 	apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"k8s.io/client-go/rest"
@@ -46,7 +56,7 @@ const (
 	applicationPath = "../config/samples/app_v1beta1_application.yaml"
 )
 
-var _ = Describe("Application CRD should install correctly", func() {
+var _ = Describe("Application CRD e2e", func() {
 	s := scheme.Scheme
 	_ = appv1beta1.AddToScheme(s)
 
@@ -65,6 +75,12 @@ var _ = Describe("Application CRD should install correctly", func() {
 		log.Fatal("Unable to construct extensions client", err)
 	}
 
+	var managerStdout bytes.Buffer
+	var managerStderr bytes.Buffer
+	managerCmd := exec.Command("../bin/manager", "--sync-period", "30")
+	managerCmd.Stdout = &managerStdout
+	managerCmd.Stderr = &managerStderr
+
 	It("should create CRD", func() {
 		err = testutil.CreateCRD(extClient, crd)
 		Expect(err).NotTo(HaveOccurred())
@@ -84,8 +100,128 @@ var _ = Describe("Application CRD should install correctly", func() {
 		Expect(err).NotTo(HaveOccurred())
 	})
 
+	It("should start the controller", func() {
+		err = managerCmd.Start()
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should create the wordpress application", func() {
+		err = applyKustomize("../docs/examples/wordpress")
+		Expect(err).NotTo(HaveOccurred())
+	})
+
+	It("should update application status", func() {
+		kubeClient := getKubeClientOrDie(config, s)
+		application := &appv1beta1.Application{}
+		objectKey := types.NamespacedName{
+			Namespace: metav1.NamespaceDefault,
+			Name:      "wordpress-01",
+		}
+		waitForApplicationStatusUpdated(kubeClient, objectKey, application)
+		Expect(application.Status.ObservedGeneration).To(BeNumerically("<=", 5))
+		Expect(application.Status.ComponentList.Objects).To(HaveLen(5))
+	})
+
+	It("should add ownerReference to components", func() {
+		kubeClient := getKubeClientOrDie(config, s)
+		matchingLabels := map[string]string{"app.kubernetes.io/name": "wordpress-01"}
+
+		list := &unstructured.UnstructuredList{}
+		list.SetGroupVersionKind(schema.GroupVersionKind{
+			Group: "",
+			Kind:  "Service",
+		})
+		validateComponentOwnerReferences(kubeClient, list, matchingLabels)
+
+		list.SetGroupVersionKind(schema.GroupVersionKind{
+			Group: "apps",
+			Kind:  "StatefulSet",
+		})
+		validateComponentOwnerReferences(kubeClient, list, matchingLabels)
+	})
+
+	It("should stop the controller", func() {
+		err = managerCmd.Process.Signal(os.Interrupt)
+		_, _ = io.Copy(os.Stderr, &managerStderr)
+		_, _ = io.Copy(os.Stdout, &managerStdout)
+		Expect(err).NotTo(HaveOccurred())
+	})
+
 	It("should delete application CRD", func() {
 		err = testutil.DeleteCRD(extClient, crd.Name)
 		Expect(err).NotTo(HaveOccurred())
 	})
 })
+
+func validateComponentOwnerReferences(kubeClient client.Client, list *unstructured.UnstructuredList, matchedingLabels map[string]string) {
+	componentsUpdated := false
+	_ = wait.PollImmediate(time.Second, time.Second*30, func() (bool, error) {
+
+		if err := kubeClient.List(context.TODO(), list, client.InNamespace(metav1.NamespaceDefault), client.MatchingLabels(matchedingLabels)); err != nil {
+			return false, err
+		}
+
+		updated := true
+		for _, item := range list.Items {
+			if item.GetOwnerReferences() == nil || len(item.GetOwnerReferences()) < 1 || item.GetOwnerReferences()[0].Name != "wordpress-01" {
+				updated = false
+			}
+		}
+		componentsUpdated = updated
+		return updated, nil
+	})
+	Expect(componentsUpdated).To(BeTrue())
+}
+
+func waitForApplicationStatusUpdated(kubeClient client.Client, key client.ObjectKey, app *appv1beta1.Application) {
+	_ = wait.PollImmediate(time.Second, time.Second*180, func() (bool, error) {
+		if err := kubeClient.Get(context.TODO(), key, app); err != nil {
+			return false, err
+		}
+
+		if app.Status.ComponentList.Objects != nil && len(app.Status.ComponentList.Objects) == 5 && app.Status.Conditions != nil {
+			return true, nil
+		}
+		return false, nil
+	})
+}
+
+func applyKustomize(path string) error {
+	var err error
+	var kubectlOP bytes.Buffer
+	var kubectlError bytes.Buffer
+	var kustError bytes.Buffer
+
+	kustomize := exec.Command("../hack/tools/bin/kustomize", "build", path)
+	kubectl := exec.Command("../hack/tools/bin/kubectl", "apply", "-f", "-")
+
+	r, w := io.Pipe()
+	kustomize.Stdout = w
+	kustomize.Stderr = &kustError
+	kubectl.Stdin = r
+	kubectl.Stderr = &kubectlError
+	kubectl.Stdout = &kubectlOP
+
+	err = kustomize.Start()
+	if err != nil {
+		return err
+	}
+	err = kubectl.Start()
+	if err != nil {
+		return err
+	}
+	err = kustomize.Wait()
+	if err != nil {
+		_, _ = io.Copy(os.Stdout, &kustError)
+		return err
+	}
+	w.Close()
+	err = kubectl.Wait()
+	if err != nil {
+		_, _ = io.Copy(os.Stdout, &kubectlError)
+		return err
+	}
+	_, _ = io.Copy(os.Stdout, &kubectlOP)
+
+	return nil
+}

e2e/test_e2e.sh → e2e/run_e2e_test.sh

@@ -1,24 +1,27 @@
-#!/usr/bin/env bash
+#!/bin/bash -x
 # Copyright 2020 The Kubernetes Authors.
 # SPDX-License-Identifier: Apache-2.0
 
-
-
 set -o errexit
 set -o nounset
 set -o pipefail
 
-K8S_VERSION="v1.16.4"
-
-export KUBECONFIG="$(${BIN}/kind get kubeconfig-path --name="kind")"
-# image supported version are listed at https://hub.docker.com/r/kindest/node/tags
-${BIN}/kind create cluster -v 4 --retain --wait=1m --config e2e/kind-config.yaml --image=kindest/node:$K8S_VERSION
-
 # remove running containers on exit
 function cleanup() {
     ${BIN}/kind delete cluster
+    echo done
 }
 
 trap cleanup EXIT
 
+K8S_VERSION="v1.16.4"
+
+export KUBECONFIG="$(${BIN}/kind get kubeconfig-path --name="kind")"
+# image supported version are listed at https://hub.docker.com/r/kindest/node/tags
+${BIN}/kind create cluster -v 4 --retain --wait=1m --config e2e/kind-config.yaml --image=kindest/node:$K8S_VERSION
+
+echo $KUBECONFIG
+cat $KUBECONFIG
+
+# Test CRD installation and Application object creation
 go test -v ./e2e/main_test.go