[ingress-nginx] Improve nginx failover

modules/402-ingress-nginx/images/proxy-failover-iptables/rootfs/usr/bin/iptables-loop

@@ -22,7 +22,8 @@ SOCKET_EXISTS_RULE="-m socket --nowildcard -m mark --mark 0 -j RETURN"
 MARK_HTTP_RULE="-p tcp --dport 80 -j MARK --set-mark 1"
 MARK_HTTPS_RULE="-p tcp --dport 443 -j MARK --set-mark 1"
 SAVE_MARK_RULE="-j CONNMARK --save-mark"
-RESTORE_MARK_RULE="-j CONNMARK --restore-mark"
+RESTORE_HTTP_MARK_RULE="-p tcp --dport 80 -j CONNMARK --restore-mark"
+RESTORE_HTTPS_MARK_RULE="-p tcp --dport 443 -j CONNMARK --restore-mark"
 DNAT_HTTP_RULE="-p tcp --dport 80 -j DNAT --to-destination 127.0.0.1:81"
 DNAT_HTTPS_RULE="-p tcp --dport 443 -j DNAT --to-destination 127.0.0.1:444"
 INPUT_ACCEPT_RULE="-p tcp -m multiport --dport 81,444 -d 127.0.0.1 -m comment --comment ingress-failover -j ACCEPT"
@@ -36,15 +37,18 @@ iptables -w -t nat -C PREROUTING $JUMP_RULE >/dev/null 2>&1 && iptables -w -t na
 iptables -w -t nat -n --list "$CHAIN_NAME" >/dev/null 2>&1 || iptables -w -t nat -N "$CHAIN_NAME"
 iptables -w -C INPUT $INPUT_ACCEPT_RULE >/dev/null 2>&1 || iptables -w -I INPUT 1 $INPUT_ACCEPT_RULE
 
+# restore conn mark
+iptables -w -t mangle -C PREROUTING $RESTORE_HTTP_MARK_RULE >/dev/null 2>&1 || iptables -w -t mangle -I PREROUTING 1 $RESTORE_HTTP_MARK_RULE
+iptables -w -t mangle -C PREROUTING $RESTORE_HTTPS_MARK_RULE >/dev/null 2>&1 || iptables -w -t mangle -I PREROUTING 2 $RESTORE_HTTPS_MARK_RULE
+
 # flush and fill ingress-failover chain
 iptables -w -t nat -F "$CHAIN_NAME"
-iptables -w -t nat -I "$CHAIN_NAME" 1 $RESTORE_MARK_RULE
-iptables -w -t nat -I "$CHAIN_NAME" 2 $SOCKET_EXISTS_RULE
-iptables -w -t nat -I "$CHAIN_NAME" 3 $MARK_HTTP_RULE
-iptables -w -t nat -I "$CHAIN_NAME" 4 $MARK_HTTPS_RULE
-iptables -w -t nat -I "$CHAIN_NAME" 5 $SAVE_MARK_RULE
-iptables -w -t nat -I "$CHAIN_NAME" 6 $DNAT_HTTP_RULE
-iptables -w -t nat -I "$CHAIN_NAME" 7 $DNAT_HTTPS_RULE
+iptables -w -t nat -I "$CHAIN_NAME" 1 $SOCKET_EXISTS_RULE
+iptables -w -t nat -I "$CHAIN_NAME" 2 $MARK_HTTP_RULE
+iptables -w -t nat -I "$CHAIN_NAME" 3 $MARK_HTTPS_RULE
+iptables -w -t nat -I "$CHAIN_NAME" 4 $SAVE_MARK_RULE
+iptables -w -t nat -I "$CHAIN_NAME" 5 $DNAT_HTTP_RULE
+iptables -w -t nat -I "$CHAIN_NAME" 6 $DNAT_HTTPS_RULE
 
 # restore jump-rule
 iptables -w -t nat -C PREROUTING $JUMP_RULE >/dev/null 2>&1 || iptables -w -t nat -I PREROUTING 1 $JUMP_RULE
@@ -57,9 +61,9 @@ while true; do
   code=$(wget --server-response http://127.0.0.1:10254/healthz 2>&1 | awk '/^  HTTP/{print $2}')
 
   if [ -n "$code" ] && [ "$code" -eq 200 ]; then
-    iptables -w -t nat -C "$CHAIN_NAME" $SOCKET_EXISTS_RULE >/dev/null 2>&1 || iptables -w -t nat -I "$CHAIN_NAME" 2 $SOCKET_EXISTS_RULE
+    iptables -w -t nat -C "$CHAIN_NAME" $SOCKET_EXISTS_RULE >/dev/null 2>&1 || iptables -w -t nat -I "$CHAIN_NAME" 1 $SOCKET_EXISTS_RULE
   else
-    iptables -w -t nat -C "$CHAIN_NAME" $SOCKET_EXISTS_RULE >/dev/null 2>&1 && iptables -w -t nat -D "$CHAIN_NAME" 2
+    iptables -w -t nat -C "$CHAIN_NAME" $SOCKET_EXISTS_RULE >/dev/null 2>&1 && iptables -w -t nat -D "$CHAIN_NAME" 1
   fi
 
   sleep 15

modules/402-ingress-nginx/images/proxy-failover/rootfs/etc/nginx/nginx.conf.tpl

@@ -46,12 +46,12 @@ stream {
   }
 
   server {
-    listen 127.0.0.1:81;
+    listen 127.0.0.1:81 so_keepalive=off reuseport;
     proxy_pass http;
   }
 
   server {
-    listen 127.0.0.1:444;
+    listen 127.0.0.1:444 so_keepalive=off reuseport;
     proxy_pass https;
   }
 }

modules/402-ingress-nginx/template_tests/controller_test.go

@@ -183,7 +183,7 @@ memory: 200Mi`))
 			Expect(hec.KubernetesResource("HorizontalPodAutoscaler", "d8-ingress-nginx", "hpa-scaler-test-without-hpa").Exists()).To(BeFalse())
 			Expect(hec.KubernetesResource("DaemonSet", "d8-ingress-nginx", "controller-test-without-hpa").Exists()).To(BeTrue())
 
-			Expect(hec.KubernetesResource("PrometheusRule", "d8-monitoring", "prometheus-metrics-adapter-d8-ingress-nginx-cpu-utlization-for-hpa").Exists()).To(BeTrue())
+			Expect(hec.KubernetesResource("PrometheusRule", "d8-monitoring", "prometheus-metrics-adapter-d8-ingress-nginx-cpu-utilization-for-hpa").Exists()).To(BeTrue())
 			Expect(hec.KubernetesResource("ConfigMap", "d8-ingress-nginx", "test-lbwpp-config").Exists()).To(BeTrue())
 			Expect(hec.KubernetesResource("ConfigMap", "d8-ingress-nginx", "test-lbwpp-custom-headers").Exists()).To(BeTrue())
 			Expect(hec.KubernetesResource("Secret", "d8-ingress-nginx", "ingress-nginx-test-lbwpp-auth-tls").Exists()).To(BeTrue())

modules/402-ingress-nginx/templates/controller/hpa.yaml

@@ -2,14 +2,14 @@
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
-  name: prometheus-metrics-adapter-d8-ingress-nginx-cpu-utlization-for-hpa
+  name: prometheus-metrics-adapter-d8-ingress-nginx-cpu-utilization-for-hpa
   namespace: d8-monitoring
 {{ include "helm_lib_module_labels" (list . (dict "prometheus" "main" "component" "rules")) | indent 2 }}
 spec:
   groups:
     - name: prometheus-metrics-adapter.d8-ingress-nginx
       rules:
-        - record: kube_adapter_metric_d8_ingress_nginx_ds_cpu_utlization
+        - record: kube_adapter_metric_d8_ingress_nginx_ds_cpu_utilization
           expr: sum by (controller_name) (kube_controller_pod{controller_type="DaemonSet", namespace="d8-ingress-nginx"} * on (pod) group_right(controller_name) round(rate(container_cpu_usage_seconds_total{container!="POD"}[1m]) * 100))
 
 
@@ -36,7 +36,7 @@ spec:
     - type: External
       external:
         metric:
-          name: d8_ingress_nginx_ds_cpu_utlization
+          name: d8_ingress_nginx_ds_cpu_utilization
           selector:
             matchLabels:
               controller_name: controller-{{ $name }}

modules/402-ingress-nginx/templates/failover/daemonset.yaml

@@ -70,6 +70,11 @@ spec:
       hostNetwork: true
       dnsPolicy: ClusterFirstWithHostNet
       terminationGracePeriodSeconds: 300
+      volumes:
+        - name: xtables-lock
+          hostPath:
+            path: /run/xtables.lock
+            type: FileOrCreate
       containers:
       - image: {{ $.Values.global.modulesImages.registry }}:{{ $.Values.global.modulesImages.tags.ingressNginx.proxyFailover }}
         name: nginx
@@ -96,6 +101,10 @@ spec:
             add:
             - NET_RAW
             - NET_ADMIN
+        volumeMounts:
+          - mountPath: /run/xtables.lock
+            name: xtables-lock
+            readOnly: false
       - name: nginx-exporter
         image: {{ $.Values.global.modulesImages.registry }}:{{ $.Values.global.modulesImages.tags.ingressNginx.nginxExporter }}
         args:

modules/402-ingress-nginx/webhooks/validating/ingressnginxcontroller

@@ -54,14 +54,6 @@ EOF
   fi
 
 
-  # Don't allow VPA mode for LoadBalancer ingress controller
-  if context::jq -e -r '.review.request.object | ((.spec.inlet == "LoadBalancer" or .spec.inlet == "LoadBalancerWithProxyProtocol" ) and (.spec.resourcesRequests.mode == "VPA"))' >/dev/null 2>&1; then
-    cat <<EOF > $VALIDATING_RESPONSE_PATH
-{"allowed":false, "message":"'VPA' resource requests managing (.spec.resourcesRequests.mode) is not allowed for LoadBalancer IngressNginxController"}
-EOF
-    exit 0;
-  fi
-
   # allowed response
   cat <<EOF > $VALIDATING_RESPONSE_PATH
 {"allowed":true}