From 74e195962840ba811bd8eb2938aaffac95e54411 Mon Sep 17 00:00:00 2001 From: Ricardo Katz Date: Sun, 28 Apr 2024 23:51:31 +0000 Subject: [PATCH] Try and miserably fail to make this thing work --- .../old/controller-deploymentOLD.old | 246 ++++++++++++++++++ .../templates/controller-deployment.yaml | 70 ++++- charts/ingress-nginx/values.yaml | 6 +- internal/ingress/controller/nginx.go | 10 + 4 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 charts/ingress-nginx/old/controller-deploymentOLD.old diff --git a/charts/ingress-nginx/old/controller-deploymentOLD.old b/charts/ingress-nginx/old/controller-deploymentOLD.old new file mode 100644 index 00000000000..879802779e9 --- /dev/null +++ b/charts/ingress-nginx/old/controller-deploymentOLD.old @@ -0,0 +1,246 @@ +{{- if eq .Values.controller.kind "Deployment" -}} +{{- include "isControllerTagValid" . -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + {{- include "ingress-nginx.labels" . | nindent 4 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ include "ingress-nginx.controller.fullname" . }} + namespace: {{ include "ingress-nginx.namespace" . }} + {{- if .Values.controller.annotations }} + annotations: {{ toYaml .Values.controller.annotations | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: + {{- include "ingress-nginx.selectorLabels" . | nindent 6 }} + app.kubernetes.io/component: controller + {{- if eq .Values.controller.autoscaling.enabled .Values.controller.keda.enabled }} + replicas: {{ .Values.controller.replicaCount }} + {{- end }} + revisionHistoryLimit: {{ .Values.revisionHistoryLimit }} + {{- if .Values.controller.updateStrategy }} + strategy: {{ toYaml .Values.controller.updateStrategy | nindent 4 }} + {{- end }} + minReadySeconds: {{ .Values.controller.minReadySeconds }} + template: + metadata: + {{- if .Values.controller.podAnnotations }} + annotations: + {{- range $key, $value := .Values.controller.podAnnotations }} + {{ $key }}: {{ $value | quote }} + {{- end }} + {{- end }} + labels: + {{- include "ingress-nginx.labels" . | nindent 8 }} + app.kubernetes.io/component: controller + {{- with .Values.controller.labels }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.controller.podLabels }} + {{- toYaml .Values.controller.podLabels | nindent 8 }} + {{- end }} + spec: + {{- if .Values.controller.dnsConfig }} + dnsConfig: {{ toYaml .Values.controller.dnsConfig | nindent 8 }} + {{- end }} + {{- if .Values.controller.hostAliases }} + hostAliases: {{ tpl (toYaml .Values.controller.hostAliases) $ | nindent 8 }} + {{- end }} + {{- if .Values.controller.hostname }} + hostname: {{ toYaml .Values.controller.hostname | nindent 8 }} + {{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: {{ toYaml .Values.imagePullSecrets | nindent 8 }} + {{- end }} + {{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName | quote }} + {{- end }} + {{- if or .Values.controller.podSecurityContext .Values.controller.sysctls }} + securityContext: + {{- if .Values.controller.podSecurityContext }} + {{- toYaml .Values.controller.podSecurityContext | nindent 8 }} + {{- end }} + {{- if .Values.controller.sysctls }} + sysctls: + {{- range $sysctl, $value := .Values.controller.sysctls }} + - name: {{ $sysctl | quote }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.controller.shareProcessNamespace }} + shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }} + {{- end }} + containers: + - name: {{ .Values.controller.containerName }} + {{- with .Values.controller.image }} + image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{ end }}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }} + {{- end }} + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + {{- if .Values.controller.lifecycle }} + lifecycle: {{ toYaml .Values.controller.lifecycle | nindent 12 }} + {{- end }} + args: {{ include "ingress-nginx.params" . | nindent 12 }} + securityContext: {{ include "ingress-nginx.controller.containerSecurityContext" . | nindent 12 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.controller.enableMimalloc }} + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + {{- end }} + {{- if .Values.controller.extraEnvs }} + {{- toYaml .Values.controller.extraEnvs | nindent 12 }} + {{- end }} + {{- if .Values.controller.startupProbe }} + startupProbe: {{ toYaml .Values.controller.startupProbe | nindent 12 }} + {{- end }} + {{- if .Values.controller.livenessProbe }} + livenessProbe: {{ toYaml .Values.controller.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.controller.readinessProbe }} + readinessProbe: {{ toYaml .Values.controller.readinessProbe | nindent 12 }} + {{- end }} + ports: + {{- range $key, $value := .Values.controller.containerPort }} + - name: {{ $key }} + containerPort: {{ $value }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ index $.Values.controller.hostPort.ports $key | default $value }} + {{- end }} + {{- end }} + {{- if .Values.controller.metrics.enabled }} + - name: {{ .Values.controller.metrics.portName }} + containerPort: {{ .Values.controller.metrics.port }} + protocol: TCP + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook + containerPort: {{ .Values.controller.admissionWebhooks.port }} + protocol: TCP + {{- end }} + {{- range $key, $value := .Values.tcp }} + - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-tcp + containerPort: {{ $key }} + protocol: TCP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- range $key, $value := .Values.udp }} + - name: {{ if $.Values.portNamePrefix }}{{ $.Values.portNamePrefix }}-{{ end }}{{ $key }}-udp + containerPort: {{ $key }} + protocol: UDP + {{- if $.Values.controller.hostPort.enabled }} + hostPort: {{ $key }} + {{- end }} + {{- end }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} + volumeMounts: + {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} + - name: modules + {{- if .Values.controller.image.chroot }} + mountPath: /chroot/modules_mount + {{- else }} + mountPath: /modules_mount + {{- end }} + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - mountPath: /etc/nginx/template + name: nginx-template-volume + readOnly: true + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + mountPath: /usr/local/certificates/ + readOnly: true + {{- end }} + {{- if .Values.controller.extraVolumeMounts }} + {{- toYaml .Values.controller.extraVolumeMounts | nindent 12 }} + {{- end }} + {{- end }} + {{- if .Values.controller.resources }} + resources: {{ toYaml .Values.controller.resources | nindent 12 }} + {{- end }} + {{- if .Values.controller.extraContainers }} + {{- toYaml .Values.controller.extraContainers | nindent 8 }} + {{- end }} + {{- if (or .Values.controller.extraInitContainers .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} + initContainers: + {{- if .Values.controller.extraInitContainers }} + {{- toYaml .Values.controller.extraInitContainers | nindent 8 }} + {{- end }} + {{- if .Values.controller.extraModules }} + {{- range .Values.controller.extraModules }} + {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }} + {{- include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }} + {{- end }} + {{- end }} + {{- if .Values.controller.opentelemetry.enabled }} + {{- with .Values.controller.opentelemetry }} + {{- $containerSecurityContext := .containerSecurityContext | default $.Values.controller.containerSecurityContext }} + {{- include "extraModules" (dict "name" .name "image" .image "containerSecurityContext" $containerSecurityContext "resources" .resources) | nindent 8 }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.controller.hostNetwork }} + hostNetwork: {{ .Values.controller.hostNetwork }} + {{- end }} + {{- if .Values.controller.nodeSelector }} + nodeSelector: {{ toYaml .Values.controller.nodeSelector | nindent 8 }} + {{- end }} + {{- if .Values.controller.tolerations }} + tolerations: {{ toYaml .Values.controller.tolerations | nindent 8 }} + {{- end }} + {{- if .Values.controller.affinity }} + affinity: {{ toYaml .Values.controller.affinity | nindent 8 }} + {{- end }} + {{- if .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: {{ tpl (toYaml .Values.controller.topologySpreadConstraints) $ | nindent 8 }} + {{- end }} + serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} + volumes: + {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled)}} + - name: modules + emptyDir: {} + {{- end }} + {{- if .Values.controller.customTemplate.configMapName }} + - name: nginx-template-volume + configMap: + name: {{ .Values.controller.customTemplate.configMapName }} + items: + - key: {{ .Values.controller.customTemplate.configMapKey }} + path: nginx.tmpl + {{- end }} + {{- if .Values.controller.admissionWebhooks.enabled }} + - name: webhook-cert + secret: + secretName: {{ include "ingress-nginx.admissionWebhooks.fullname" . }} + {{- if .Values.controller.admissionWebhooks.certManager.enabled }} + items: + - key: tls.crt + path: cert + - key: tls.key + path: key + {{- end }} + {{- end }} + {{- if .Values.controller.extraVolumes }} + {{ toYaml .Values.controller.extraVolumes | nindent 8 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/ingress-nginx/templates/controller-deployment.yaml b/charts/ingress-nginx/templates/controller-deployment.yaml index 879802779e9..7c276c5bbdc 100644 --- a/charts/ingress-nginx/templates/controller-deployment.yaml +++ b/charts/ingress-nginx/templates/controller-deployment.yaml @@ -78,6 +78,59 @@ spec: shareProcessNamespace: {{ .Values.controller.shareProcessNamespace }} {{- end }} containers: + - args: + - /nginx-ingress-dataplane + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + image: ingress-controller/dataplane:1.0.0-dev + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + name: "dataplane" + ports: + - containerPort: 80 + hostPort: 80 + name: http + protocol: TCP + - containerPort: 443 + hostPort: 443 + name: https + protocol: TCP + resources: + requests: + cpu: 100m + memory: 90Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /etc/ingress-controller + name: ingress-controller + - mountPath: /etc/nginx/conf + name: nginx-conf + - mountPath: /var/run/secrets/kubernetes.io/serviceaccount + name: emptysecret - name: {{ .Values.controller.containerName }} {{- with .Values.controller.image }} image: {{ if .repository }}{{ .repository }}{{ else }}{{ .registry }}/{{ include "ingress-nginx.image" . }}{{ end }}:{{ .tag }}{{ include "ingress-nginx.imageDigest" . }} @@ -148,8 +201,12 @@ spec: hostPort: {{ $key }} {{- end }} {{- end }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} volumeMounts: + - mountPath: /etc/ingress-controller + name: ingress-controller + - mountPath: /etc/nginx/conf + name: nginx-conf + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} - name: modules {{- if .Values.controller.image.chroot }} @@ -213,8 +270,17 @@ spec: {{- end }} serviceAccountName: {{ template "ingress-nginx.serviceAccountName" . }} terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} - {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} volumes: + - name: ingress-controller + emptyDir: + sizeLimit: 500Mi + - name: nginx-conf + emptyDir: + sizeLimit: 500Mi + - name: emptysecret + emptyDir: + sizeLimit: 1Mi + {{- if (or .Values.controller.customTemplate.configMapName .Values.controller.extraVolumeMounts .Values.controller.admissionWebhooks.enabled .Values.controller.extraVolumes .Values.controller.extraModules .Values.controller.opentelemetry.enabled) }} {{- if (or .Values.controller.extraModules .Values.controller.opentelemetry.enabled)}} - name: modules emptyDir: {} diff --git a/charts/ingress-nginx/values.yaml b/charts/ingress-nginx/values.yaml index 85990614b30..5a9a80cbd2d 100644 --- a/charts/ingress-nginx/values.yaml +++ b/charts/ingress-nginx/values.yaml @@ -42,9 +42,9 @@ controller: # -- Configures the controller container name containerName: controller # -- Configures the ports that the nginx-controller listens on - containerPort: - http: 80 - https: 443 + # containerPort: + # http: 80 + # https: 443 # -- Global configuration passed to the ConfigMap consumed by the controller. Values may contain Helm templates. # Ref.: https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/ config: {} diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 6bc00daeb45..5c32f7d4414 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -44,6 +44,7 @@ import ( v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/flowcontrol" + "k8s.io/client-go/util/retry" "k8s.io/ingress-nginx/pkg/tcpproxy" adm_controller "k8s.io/ingress-nginx/internal/admission/controller" @@ -190,6 +191,15 @@ func NewNGINXController(config *Configuration, mc metric.Collector) *NGINXContro // everything. // It should be guaranteed by the dataplane that this file is changed just once // NGINX finishes starting + err = retry.OnError(retry.DefaultBackoff, func(err error) bool { + return true + }, func() error { + _, errStat := os.Stat(nginxdataplane.ReadyFile) + return errStat + }) + if err != nil { + klog.Fatalf("error waiting for ready file: %s", err) + } _, err = file.NewFileWatcherUpdateOnly(nginxdataplane.ReadyFile, true, func() { klog.Fatalf("readiness file changed, restarting contorller") })