Try and miserably fail to make this thing work

charts/ingress-nginx/old/controller-deploymentOLD.old

@@ -0,0 +1,246 @@
+{{- if eq .Values.controller.kind "Deployment" -}}
+{{- include  "isControllerTagValid" . -}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    {{- include "ingress-nginx.labels" . | nindent 4 }}
+    app.kubernetes.io/component: controller
+    {{- with .Values.controller.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+  name: {{ include "ingress-nginx.controller.fullname" . }}
+  namespace: {{ include "ingress-nginx.namespace" . }}
+  {{- if .Values.controller.annotations }}
+  annotations: {{ toYaml .Values.controller.annotations | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels:
+      {{- include "ingress-nginx.selectorLabels" . | nindent 6 }}
+      app.kubernetes.io/component: controller
+  {{- if eq .Values.controller.autoscaling.enabled .Values.controller.keda.enabled }}
+  replicas: {{ .Values.controller.replicaCount }}
+  {{- end }}
+  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
+  {{- if .Values.controller.updateStrategy }}
+  strategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }}
+  {{- end }}
+  minReadySeconds: {{ .Values.controller.minReadySeconds }}
+  template:
+    metadata:
+    {{- if .Values.controller.podAnnotations }}
+      annotations:
+      {{- range $key, $value := .Values.controller.podAnnotations }}
+        {{ $key }}: {{ $value | quote }}
+      {{- end }}
+    {{- end }}
+      labels:
+        {{- include "ingress-nginx.labels" . | nindent 8 }}
+        app.kubernetes.io/component: controller
+        {{- with .Values.controller.labels }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
+      {{- if .Values.controller.podLabels }}
+        {{- toYaml .Values.controller.podLabels | nindent 8 }}
+      {{- end }}
+    spec:
+    {{- if .Values.controller.dnsConfig }}
+      dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }}
+    {{- end }}
+    {{- if .Values.controller.hostAliases }}
+      hostAliases: {{ tpl (toYaml .Values.controller.hostAliases) $ | nindent 8 }}
+    {{- end }}
+    {{- if .Values.controller.hostname }}
+      hostname: {{ toYaml .Values.controller.hostname | nindent 8 }}
+    {{- end }}
+      dnsPolicy: {{ .Values.controller.dnsPolicy }}
+    {{- if .Values.imagePullSecrets }}
+      imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }}
+    {{- end }}
+    {{- if .Values.controller.priorityClassName }}
+      priorityClassName: {{ .Values.controller.priorityClassName | quote }}
+    {{- end }}
+    {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }}
+      securityContext:
+      {{- if .Values.controller.podSecurityContext }}
+        {{- toYaml .Values.controller.podSecurityContext | nindent 8 }}
+      {{- end }}
+      {{- if .Values.controller.sysctls }}
+        sysctls:
+        {{- range $sysctl, $value := .Values.controller.sysctls }}
+          - name: {{ $sysctl | quote }}
+            value: {{ $value | quote }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
+    {{- if .Values.controller.shareProcessNamespace }}
+      shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }}
+    {{- end }}
+      containers:
+        - name: {{ .Values.controller.containerName }}
+          {{- with .Values.controller.image }}
+          image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{ end }}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}
+          {{- end }}
+          imagePullPolicy: {{ .Values.controller.image.pullPolicy }}
+        {{- if .Values.controller.lifecycle }}
+          lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }}
+        {{- end }}
+          args: {{ include "ingress-nginx.params" . | nindent 12 }}
+          securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }}
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          {{- if .Values.controller.enableMimalloc }}
+            - name: LD_PRELOAD
+              value: /usr/local/lib/libmimalloc.so
+          {{- end }}
+          {{- if .Values.controller.extraEnvs }}
+            {{- toYaml .Values.controller.extraEnvs | nindent 12 }}
+          {{- end }}
+          {{- if .Values.controller.startupProbe }}
+          startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }}
+          {{- end }}
+          {{- if .Values.controller.livenessProbe }}
+          livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }}
+          {{- end }}
+          {{- if .Values.controller.readinessProbe }}
+          readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }}
+          {{- end }}
+          ports:
+          {{- range $key, $value := .Values.controller.containerPort }}
+            - name: {{ $key }}
+              containerPort: {{ $value }}
+              protocol: TCP
+              {{- if $.Values.controller.hostPort.enabled }}
+              hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }}
+              {{- end }}
+          {{- end }}
+          {{- if .Values.controller.metrics.enabled }}
+            - name: {{ .Values.controller.metrics.portName }}
+              containerPort: {{ .Values.controller.metrics.port }}
+              protocol: TCP
+          {{- end }}
+          {{- if .Values.controller.admissionWebhooks.enabled }}
+            - name: webhook
+              containerPort: {{ .Values.controller.admissionWebhooks.port }}
+              protocol: TCP
+          {{- end }}
+          {{- range $key, $value := .Values.tcp }}
+            - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp
+              containerPort: {{ $key }}
+              protocol: TCP
+              {{- if $.Values.controller.hostPort.enabled }}
+              hostPort: {{ $key }}
+              {{- end }}
+          {{- end }}
+          {{- range $key, $value := .Values.udp }}
+            - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp
+              containerPort: {{ $key }}
+              protocol: UDP
+              {{- if $.Values.controller.hostPort.enabled }}
+              hostPort: {{ $key }}
+              {{- end }}
+          {{- end }}
+        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
+          volumeMounts:
+          {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
+            - name: modules
+            {{- if .Values.controller.image.chroot }}
+              mountPath: /chroot/modules_mount
+            {{- else }}
+              mountPath: /modules_mount
+            {{- end }}
+          {{- end }}
+          {{- if .Values.controller.customTemplate.configMapName }}
+            - mountPath: /etc/nginx/template
+              name: nginx-template-volume
+              readOnly: true
+          {{- end }}
+          {{- if .Values.controller.admissionWebhooks.enabled }}
+            - name: webhook-cert
+              mountPath: /usr/local/certificates/
+              readOnly: true
+          {{- end }}
+          {{- if .Values.controller.extraVolumeMounts }}
+            {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }}
+          {{- end }}
+        {{- end }}
+        {{- if .Values.controller.resources }}
+          resources: {{ toYaml .Values.controller.resources | nindent 12 }}
+        {{- end }}
+      {{- if .Values.controller.extraContainers }}
+        {{- toYaml .Values.controller.extraContainers | nindent 8 }}
+      {{- end }}
+    {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
+      initContainers:
+      {{- if .Values.controller.extraInitContainers }}
+        {{- toYaml .Values.controller.extraInitContainers | nindent 8 }}
+      {{- end }}
+      {{- if .Values.controller.extraModules }}
+        {{- range .Values.controller.extraModules }}
+          {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }}
+          {{- include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      {{- if .Values.controller.opentelemetry.enabled }}
+        {{- with .Values.controller.opentelemetry }}
+          {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }}
+          {{- include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    {{- end }}
+    {{- if .Values.controller.hostNetwork }}
+      hostNetwork: {{ .Values.controller.hostNetwork }}
+    {{- end }}
+    {{- if .Values.controller.nodeSelector }}
+      nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }}
+    {{- end }}
+    {{- if .Values.controller.tolerations }}
+      tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }}
+    {{- end }}
+    {{- if .Values.controller.affinity }}
+      affinity: {{ toYaml .Values.controller.affinity | nindent 8 }}
+    {{- end }}
+    {{- if .Values.controller.topologySpreadConstraints }}
+      topologySpreadConstraints: {{ tpl (toYaml .Values.controller.topologySpreadConstraints) $ | nindent 8 }}
+    {{- end }}
+      serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
+      terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
+    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
+      volumes:
+      {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled)}}
+        - name: modules
+          emptyDir: {}
+      {{- end }}
+      {{- if .Values.controller.customTemplate.configMapName }}
+        - name: nginx-template-volume
+          configMap:
+            name: {{ .Values.controller.customTemplate.configMapName }}
+            items:
+            - key: {{ .Values.controller.customTemplate.configMapKey }}
+              path: nginx.tmpl
+      {{- end }}
+      {{- if .Values.controller.admissionWebhooks.enabled }}
+        - name: webhook-cert
+          secret:
+            secretName: {{ include "ingress-nginx.admissionWebhooks.fullname" . }}
+        {{- if .Values.controller.admissionWebhooks.certManager.enabled }}
+            items:
+              - key: tls.crt
+                path: cert
+              - key: tls.key
+                path: key
+        {{- end }}
+      {{- end }}
+      {{- if .Values.controller.extraVolumes }}
+        {{ toYaml .Values.controller.extraVolumes | nindent 8 }}
+      {{- end }}
+    {{- end }}
+{{- end }}

charts/ingress-nginx/templates/controller-deployment.yaml

@@ -78,6 +78,59 @@ spec:
       shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }}
     {{- end }}
       containers:
+        - args:
+          - /nginx-ingress-dataplane
+          env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: LD_PRELOAD
+            value: /usr/local/lib/libmimalloc.so
+          image: ingress-controller/dataplane:1.0.0-dev
+          imagePullPolicy: IfNotPresent
+          lifecycle:
+            preStop:
+              exec:
+                command:
+                - /wait-shutdown
+          name: "dataplane"
+          ports:
+          - containerPort: 80
+            hostPort: 80
+            name: http
+            protocol: TCP
+          - containerPort: 443
+            hostPort: 443
+            name: https
+            protocol: TCP
+          resources:
+            requests:
+              cpu: 100m
+              memory: 90Mi
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              add:
+              - NET_BIND_SERVICE
+              drop:
+              - ALL
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+            runAsUser: 101
+            seccompProfile:
+              type: RuntimeDefault
+          volumeMounts:
+          - mountPath: /etc/ingress-controller
+            name: ingress-controller
+          - mountPath: /etc/nginx/conf
+            name: nginx-conf
+          - mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+            name: emptysecret
         - name: {{ .Values.controller.containerName }}
           {{- with .Values.controller.image }}
           image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{ end }}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }}
@@ -148,8 +201,12 @@ spec:
               hostPort: {{ $key }}
               {{- end }}
           {{- end }}
-        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
           volumeMounts:
+            - mountPath: /etc/ingress-controller
+              name: ingress-controller
+            - mountPath: /etc/nginx/conf
+              name: nginx-conf
+        {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
           {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
             - name: modules
             {{- if .Values.controller.image.chroot }}
@@ -213,8 +270,17 @@ spec:
     {{- end }}
       serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }}
       terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
-    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
       volumes:
+        - name: ingress-controller
+          emptyDir:
+            sizeLimit: 500Mi
+        - name: nginx-conf
+          emptyDir:
+            sizeLimit: 500Mi
+        - name: emptysecret
+          emptyDir:
+            sizeLimit: 1Mi
+    {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }}
       {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled)}}
         - name: modules
           emptyDir: {}

charts/ingress-nginx/values.yaml

@@ -42,9 +42,9 @@ controller:
   # -- Configures the controller container name
   containerName: controller
   # -- Configures the ports that the nginx-controller listens on
-  containerPort:
-    http: 80
-    https: 443
+  # containerPort:
+  #  http: 80
+  #  https: 443
   # -- Global configuration passed to the ConfigMap consumed by the controller. Values may contain Helm templates.
   # Ref.: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/
   config: {}

internal/ingress/controller/nginx.go

@@ -44,6 +44,7 @@ import (
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/flowcontrol"
+	"k8s.io/client-go/util/retry"
 	"k8s.io/ingress-nginx/pkg/tcpproxy"
 
 	adm_controller "k8s.io/ingress-nginx/internal/admission/controller"
@@ -190,6 +191,15 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro
 	// everything.
 	// It should be guaranteed by the dataplane that this file is changed just once 
 	// NGINX finishes starting
+	err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
+		return true
+	}, func() error {
+		_, errStat := os.Stat(nginxdataplane.ReadyFile)
+		return errStat
+	})
+	if err != nil {
+		klog.Fatalf("error waiting for ready file: %s", err)
+	}
 	_, err = file.NewFileWatcherUpdateOnly(nginxdataplane.ReadyFile, true, func() {
 		klog.Fatalf("readiness file changed, restarting contorller")
 	})