First pass of partial e2e

kubernetes · May 6, 2024 · 0a0d4aa · 0a0d4aa
1 parent 74e1959
commit 0a0d4aa
Show file tree

Hide file tree

Showing 13 changed files with 42 additions and 48 deletions.
diff --git a/NGINX_BASE b/NGINX_BASE
@@ -1 +1 @@
-registry.k8s.io/ingress-nginx/nginx-1.25:v0.0.6@sha256:b3e027ab191eb9461a9bcf25092eabb1d547cba164992dbd722c1aa2b4a936ee
+rpkatz/nginx-split:v1
diff --git a/build/dev-env.sh b/build/dev-env.sh
@@ -64,7 +64,7 @@ echo "[dev-env] building image"
 make build image
 docker tag "${REGISTRY}/controller:${TAG}" "${DEV_IMAGE}"
 
-export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f}
+export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245}
 
 KIND_CLUSTER_NAME="ingress-nginx-dev"
 

diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-deployment.yaml
@@ -78,27 +78,28 @@ spec:
       shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }}
     {{- end }}
       containers:
-        - args:
-          - /nginx-ingress-dataplane
-          env:
-          - name: POD_NAME
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.name
-          - name: POD_NAMESPACE
-            valueFrom:
-              fieldRef:
-                fieldPath: metadata.namespace
-          - name: LD_PRELOAD
-            value: /usr/local/lib/libmimalloc.so
+        - name: "dataplane"
           image: ingress-controller/dataplane:1.0.0-dev
           imagePullPolicy: IfNotPresent
           lifecycle:
             preStop:
               exec:
                 command:
                 - /wait-shutdown
-          name: "dataplane"
+          args:
+          - /nginx-ingress-dataplane
+          securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }}
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: LD_PRELOAD
+              value: /usr/local/lib/libmimalloc.so
           ports:
           - containerPort: 80
             hostPort: 80
@@ -112,18 +113,6 @@ spec:
             requests:
               cpu: 100m
               memory: 90Mi
-          securityContext:
-            allowPrivilegeEscalation: false
-            capabilities:
-              add:
-              - NET_BIND_SERVICE
-              drop:
-              - ALL
-            readOnlyRootFilesystem: false
-            runAsNonRoot: true
-            runAsUser: 101
-            seccompProfile:
-              type: RuntimeDefault
           volumeMounts:
           - mountPath: /etc/ingress-controller
             name: ingress-controller

diff --git a/cmd/dataplane/main.go b/cmd/dataplane/main.go
@@ -49,7 +49,8 @@ func main() {
 	}))
 
 	//mc := metric.NewDummyCollector()
-	go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort)
+	// TODO: Enable metric collector for DP again
+	//go metrics.RegisterProfiler(nginx.ProfilerAddress, nginx.ProfilerPort)
 
 	mux := http.NewServeMux()
 	metrics.RegisterHealthz(nginx.HealthPath, mux)

diff --git a/images/test-runner/Makefile b/images/test-runner/Makefile
@@ -58,7 +58,7 @@ image:
 		--build-arg CHART_TESTING_VERSION=3.8.0 \
 		--build-arg YAML_LINT_VERSION=1.33.0 \
 		--build-arg YAMALE_VERSION=4.0.4 \
-		--build-arg HELM_VERSION=3.11.2 \
+		--build-arg HELM_VERSION=3.14.3 \
 		--build-arg GINKGO_VERSION=2.17.1 \
 		--build-arg GOLINT_VERSION=latest \
 		-t ${IMAGE}:${TAG} rootfs
@@ -79,7 +79,7 @@ build: ensure-buildx
 		--build-arg CHART_TESTING_VERSION=3.8.0 \
 		--build-arg YAML_LINT_VERSION=1.33.0 \
 		--build-arg YAMALE_VERSION=4.0.4 \
-		--build-arg HELM_VERSION=3.11.2 \
+		--build-arg HELM_VERSION=3.14.3 \
 		--build-arg GINKGO_VERSION=2.17.1 \
 		--build-arg GOLINT_VERSION=latest \
 		-t ${IMAGE}:${TAG} rootfs

diff --git a/internal/ingress/controller/checker.go b/internal/ingress/controller/checker.go
@@ -19,11 +19,6 @@ package controller
 import (
 	"fmt"
 	"net/http"
-	"os"
-	"strconv"
-	"strings"
-
-	"github.com/ncabatoff/process-exporter/proc"
 
 	"k8s.io/ingress-nginx/internal/nginx"
 )
@@ -40,7 +35,8 @@ func (n *NGINXController) Check(_ *http.Request) error {
 	}
 
 	// check the nginx master process is running
-	fs, err := proc.NewFS("/proc", false)
+	// TODO: Move this checker to dataplane!
+	/*fs, err := proc.NewFS("/proc", false)
 	if err != nil {
 		return fmt.Errorf("reading /proc directory: %w", err)
 	}
@@ -58,7 +54,7 @@ func (n *NGINXController) Check(_ *http.Request) error {
 	_, err = fs.Proc(pid)
 	if err != nil {
 		return fmt.Errorf("checking for NGINX process with PID %v: %w", pid, err)
-	}
+	}*/
 
 	statusCode, _, err := nginx.NewGetStatusRequest("/is-dynamic-lb-initialized")
 	if err != nil {

diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
@@ -40,6 +40,7 @@ import (
 	"github.com/eapache/channels"
 	apiv1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes/scheme"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
@@ -191,7 +192,13 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
 	// everything.
 	// It should be guaranteed by the dataplane that this file is changed just once 
 	// NGINX finishes starting
-	err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
+	readinessBackoff := wait.Backoff{
+		Steps:    5,
+		Duration: 3 * time.Millisecond,
+		Factor:   5.0,
+		Jitter:   0.1,
+	}
+	err = retry.OnError(readinessBackoff, func(err error) bool {
 		return true
 	}, func() error {
 		_, errStat := os.Stat(nginxdataplane.ReadyFile)

diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh
@@ -24,7 +24,6 @@ E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""}
 
 reportFile="report-e2e-test-suite.xml"
 ginkgo_args=(
-  "--fail-fast"
   "--flake-attempts=2"
   "--junit-report=${reportFile}"
   "--nodes=${E2E_NODES}"

diff --git a/test/e2e/framework/exec.go b/test/e2e/framework/exec.go
@@ -67,7 +67,7 @@ func (f *Framework) ExecCommand(pod *corev1.Pod, command string) (string, error)
 	)
 
 	//nolint:gosec // Ignore G204 error
-	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container controller -- %s", KubectlPath, pod.Namespace, pod.Name, command))
+	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v exec --namespace %s %s --container dataplane -- %s", KubectlPath, pod.Namespace, pod.Name, command))
 	cmd.Stdout = &execOut
 	cmd.Stderr = &execErr
 

diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go
@@ -183,7 +183,8 @@ func (f *Framework) AfterEach() {
 		return
 	}
 
-	cmd := "cat /etc/nginx/nginx.conf"
+	// TODO: This file location should be a const
+	cmd := "cat /etc/nginx/conf/nginx.conf"
 	o, err := f.ExecCommand(f.pod, cmd)
 	if err != nil {
 		Logf("Unexpected error obtaining nginx.conf file: %v", err)
@@ -303,9 +304,9 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
 	return func() (bool, error) {
 		var cmd string
 		if name == "" {
-			cmd = "cat /etc/nginx/nginx.conf"
+			cmd = "cat /etc/nginx/conf/nginx.conf"
 		} else {
-			cmd = fmt.Sprintf("cat /etc/nginx/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name)
+			cmd = fmt.Sprintf("cat /etc/nginx/conf/nginx.conf | awk '/## start server %v/,/## end server %v/'", name, name)
 		}
 
 		o, err := f.ExecCommand(f.pod, cmd)
@@ -328,7 +329,7 @@ func (f *Framework) matchNginxConditions(name string, matcher func(cfg string) b
 
 func (f *Framework) matchNginxCustomConditions(from, to string, matcher func(cfg string) bool) wait.ConditionFunc {
 	return func() (bool, error) {
-		cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf| awk '/%v/,/%v/'", from, to)
+		cmd := fmt.Sprintf("cat /etc/nginx/conf/nginx.conf| awk '/%v/,/%v/'", from, to)
 
 		o, err := f.ExecCommand(f.pod, cmd)
 		if err != nil {

diff --git a/test/e2e/run-chart-test.sh b/test/e2e/run-chart-test.sh
@@ -62,7 +62,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
 if [ "${SKIP_CLUSTER_CREATION:-false}" = "false" ]; then
   echo "[dev-env] creating Kubernetes cluster with kind"
 
-  export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f}
+  export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245}
 
   kind create cluster \
     --verbosity=${KIND_LOG_LEVEL} \

diff --git a/test/e2e/run-kind-e2e.sh b/test/e2e/run-kind-e2e.sh
@@ -64,7 +64,7 @@ echo "Running e2e with nginx base image ${NGINX_BASE_IMAGE}"
 if [ "${SKIP_CLUSTER_CREATION}" = "false" ]; then
   echo "[dev-env] creating Kubernetes cluster with kind"
 
-  export K8S_VERSION=${K8S_VERSION:-v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f}
+  export K8S_VERSION=${K8S_VERSION:-v1.29.2@sha256:51a1434a5397193442f0be2a297b488b6c919ce8a3931be0ce822606ea5ca245}
 
   # delete the cluster if it exists
   if kind get clusters | grep "${KIND_CLUSTER_NAME}"; then

diff --git a/test/e2e/wait-for-nginx.sh b/test/e2e/wait-for-nginx.sh
@@ -34,7 +34,8 @@ function on_exit {
     test $error_code == 0 && return;
 
     echo "Obtaining ingress controller pod logs..."
-    kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE
+    kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE -c controller
+    kubectl logs -l app.kubernetes.io/name=ingress-nginx -n $NAMESPACE -c dataplane
 }
 trap on_exit EXIT