[ingress-nginx] Add 0.49 version

Signed-off-by: m.nabokikh <maksim.nabokikh@flant.com>

modules/402-ingress-nginx/crds/ingress-nginx.yaml

@@ -58,7 +58,7 @@ spec:
                     One of the supported NGINX Ingress Controller versions.
 
                     **By default**: the version in the module settings is used.
-                  enum: ['0.25', '0.26', '0.33', '0.46', '0.48']
+                  enum: ['0.25', '0.26', '0.33', '0.46', '0.48', '0.49']
                 enableIstioSidecar:
                   type: boolean
                   description: |

modules/402-ingress-nginx/images/controller-0-49/Dockerfile

@@ -0,0 +1,116 @@
+ARG BASE_ALPINE
+# controller artifact
+ARG BASE_GOLANG_16_BUSTER
+FROM $BASE_GOLANG_16_BUSTER as artifact
+WORKDIR /src/
+COPY patches/lua-info.patch /
+COPY patches/makefile.patch /
+COPY patches/healthcheck.patch /
+COPY patches/pod-ip.patch /
+ENV GOARCH=amd64
+RUN apt-get update && apt-get install -y --no-install-recommends git mercurial patch && \
+    git clone --branch controller-v0.49.1 --depth 1 https://github.com/kubernetes/ingress-nginx.git /src && \
+    patch -p1 < /lua-info.patch && \
+    patch -p1 < /makefile.patch && \
+    patch -p1 < /pod-ip.patch && \
+    patch -p1 < /healthcheck.patch && \
+    make GO111MODULE=on USE_DOCKER=false build
+
+# luarocks assets for luajit artifact
+FROM quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.26.1@sha256:d0b22f715fcea5598ef7f869d308b55289a3daaa12922fa52a1abf17703c88e7 as controller_0_26_1
+USER root
+RUN apt-get update \
+  && apt-get install -y --no-install-recommends patch gcc build-essential \
+  && luarocks install lua-protobuf 0.3.2-0 \
+  && luarocks install lua-iconv 7-3
+
+# IngressNginxController docker image
+FROM k8s.gcr.io/ingress-nginx/controller:v0.49.1@sha256:4080849490fd4f61416ad7bdba6fdd0ee58164f2e13df1128b407ce82d5f3986 as controller_0_49_1
+
+# Final image
+FROM $BASE_ALPINE
+# Based on https://github.com/kubernetes/ingress-nginx/blob/controller-v0.49.1/images/nginx/rootfs/Dockerfile
+# Based on https://github.com/kubernetes/ingress-nginx/blob/controller-v0.49.1/rootfs/Dockerfile
+ENV PATH=$PATH:/usr/local/luajit/bin:/usr/local/nginx/sbin:/usr/local/nginx/bin
+
+ENV LUA_PATH="/usr/local/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/lib/lua/?.lua;;"
+ENV LUA_CPATH="/usr/local/lib/lua/?/?.so;/usr/local/lib/lua/?.so;;"
+# Fix for openrestry luarocks paths
+ENV PATH=$PATH:/usr/local/openresty/luajit/bin:/usr/local/openresty/nginx/sbin:/usr/local/openresty/bin
+ENV LUA_PATH=$LUA_PATH;/usr/local/openresty/site/lualib/?.ljbc;/usr/local/openresty/site/lualib/?/init.ljbc;/usr/local/openresty/lualib/?.ljbc;/usr/local/openresty/lualib/?/init.ljbc;/usr/local/openresty/site/lualib/?.lua;/usr/local/openresty/site/lualib/?/init.lua;/usr/local/openresty/lualib/?.lua;/usr/local/openresty/lualib/?/init.lua;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;/usr/local/lib/lua/?.lua
+ENV LUA_CPATH=$LUA_CPATH;/usr/local/openresty/site/lualib/?.so;/usr/local/openresty/lualib/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so
+
+RUN addgroup -Sg 101 www-data \
+  && adduser -S -D -H -u 101 -h /usr/local/nginx \
+    -s /sbin/nologin -G www-data -g www-data www-data
+
+COPY --from=controller_0_49_1 /usr/local /usr/local
+COPY --from=controller_0_49_1 /opt /opt
+COPY --from=controller_0_49_1 --chown=www-data:www-data /etc /etc
+COPY --from=controller_0_49_1 --chown=www-data:www-data /dbg /dbg
+COPY --from=controller_0_49_1 --chown=www-data:www-data /nginx-ingress-controller /nginx-ingress-controller
+COPY --from=controller_0_49_1 --chown=www-data:www-data /wait-shutdown /wait-shutdown
+COPY --from=artifact /src/rootfs/bin/amd64/nginx-ingress-controller /src/rootfs/bin/amd64/dbg /
+# We use 0.26 controller image to copy lua libraries because is easier to install them on ubuntu
+COPY --from=controller_0_26_1 /usr/local/openresty/luajit /usr/local/openresty/luajit
+COPY patches/balancer-lua.patch /
+COPY patches/nginx-tpl.patch /
+COPY rootfs /
+
+RUN apk update \
+  && apk upgrade \
+  && apk add -U --no-cache \
+    bash \
+    openssl \
+    pcre \
+    zlib \
+    geoip \
+    curl ca-certificates \
+    patch \
+    yajl \
+    lmdb \
+    libxml2 \
+    libmaxminddb \
+    yaml-cpp \
+    dumb-init \
+    nano \
+    tzdata \
+    diffutils \
+    libcap \
+  && ln -s /usr/local/nginx/sbin/nginx /sbin/nginx \
+  && bash -eu -c ' \
+  writeDirs=( \
+    /var/lib/nginx/body \
+    /var/lib/nginx/fastcgi \
+    /var/lib/nginx/proxy \
+    /var/lib/nginx/scgi \
+    /var/lib/nginx/uwsgi \
+    /etc/ingress-controller \
+    /etc/ingress-controller/ssl \
+    /etc/ingress-controller/auth \
+    /var/log/audit \
+    /var/log \
+    /var/log/nginx \
+  ); \
+  for dir in "${writeDirs[@]}"; do \
+    mkdir -p ${dir}; \
+    chown -R www-data.www-data ${dir}; \
+  done' \
+  && setcap    cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller \
+  && setcap    cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+  && setcap -v cap_net_bind_service=+ep /usr/local/nginx/sbin/nginx \
+# Create symlinks to redirect nginx logs to stdout and stderr docker log collector
+  &&  ln -sf /dev/stdout /var/log/nginx/access.log \
+  && ln -sf /dev/stderr /var/log/nginx/error.log \
+  && apk del libcap \
+  && cd / \
+  && patch -p1 < /balancer-lua.patch \
+  && patch -p1 < /nginx-tpl.patch
+
+WORKDIR /
+USER www-data
+EXPOSE 80 443
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+CMD ["/nginx-ingress-controller"]
+

modules/402-ingress-nginx/images/controller-0-49/patches/README.md

@@ -0,0 +1,34 @@
+## Patches
+
+## Healthcheck
+
+After catching SIGTERM, ingress stops responding to the readiness probe.
+The combination of this patch and the `EndpointSliceTerminatingCondition` feature gate for kube-proxy helps us avoid
+traffic loss on rollout updates.
+
+Backport of the behavior of the later versions of ingress nginx controller.
+The `sleep` is needed to gracefully shut down ingress controllers behind a cloud load balancer.
+
+## Nginx TPL
+
+* Enable our metrics collector instead of the default one.
+* Enable pcre_jit.
+* Add the health check server to provide the way for an external load balancer to check that the ingress controller will be terminated soon.
+
+We do not intend to make a PR to the upstream with these changes, because there are only our custom features.
+
+## Ingress information
+
+There are two patches to fix the problem with ingress names in logs and metrics.
+Unfortunately, the PR was declined in the upstream.
+https://github.com/kubernetes/ingress-nginx/pull/4367
+
+## Pod IP
+
+This is a small patch to fix the problem with the ingress controller pod listening on 0.0.0.0 in the host network mode.
+Our PR was reverted in the upstream.
+https://github.com/kubernetes/ingress-nginx/issues/2262
+
+## Makefile
+
+Run the build locally, not inside the container.

modules/402-ingress-nginx/images/controller-0-49/patches/balancer-lua.patch

@@ -0,0 +1,45 @@
+diff --git a/rootfs/etc/nginx/lua/balancer.lua b/rootfs/etc/nginx/lua/balancer.lua
+index e9fcb32bc..d002a59bf 100644
+--- a/etc/nginx/lua/balancer.lua
++++ b/etc/nginx/lua/balancer.lua
+@@ -85,6 +85,13 @@ local function sync_backend(backend)
+
+   if not balancer then
+     balancers[backend.name] = implementation:new(backend)
++    if backend.ingress then
++      balancers[backend.name]["ingress_name"] = backend.ingress.metadata.name
++    end
++    if backend.service then
++      balancers[backend.name]["backend_name"] = backend.service.metadata.name
++      balancers[backend.name]["backend_port"] = backend.port
++    end
+     return
+   end
+
+@@ -95,6 +102,13 @@ local function sync_backend(backend)
+     ngx.log(ngx.INFO,
+       string.format("LB algorithm changed from %s to %s, resetting the instance", balancer.name, implementation.name))
+     balancers[backend.name] = implementation:new(backend)
++    if backend.ingress then
++      balancers[backend.name]["ingress_name"] = backend.ingress.metadata.name
++    end
++    if backend.service then
++      balancers[backend.name]["backend_name"] = backend.service.metadata.name
++      balancers[backend.name]["backend_port"] = backend.port
++    end
+     return
+   end
+
+@@ -229,6 +243,12 @@ function _M.balance()
+     return
+   end
+
++  if balancer["ingress_name"] then ngx.var.ingress_name = balancer["ingress_name"]:gsub("%-rwr", "") end
++  if not (ngx.var.proxy_upstream_name == "upstream-default-backend") then
++    if balancer["backend_name"] then ngx.var.service_name = balancer["backend_name"] end
++    if balancer["backend_port"] then ngx.var.service_port = balancer["backend_port"] end
++  end
++
+   local peer = balancer:balance()
+   if not peer then
+     ngx.log(ngx.WARN, "no peer was returned, balancer: " .. balancer.name)

modules/402-ingress-nginx/images/controller-0-49/patches/healthcheck.patch

@@ -0,0 +1,15 @@
+diff --git a/internal/ingress/controller/checker.go b/internal/ingress/controller/checker.go
+index 81c6e12be..772dd3c9b 100644
+--- a/internal/ingress/controller/checker.go
++++ b/internal/ingress/controller/checker.go
+@@ -36,10 +36,6 @@ func (n NGINXController) Name() string {
+
+ // Check returns if the nginx healthz endpoint is returning ok (status code 200)
+ func (n *NGINXController) Check(_ *http.Request) error {
+-	if n.isShuttingDown {
+-		return fmt.Errorf("the ingress controller is shutting down")
+-	}
+-
+ 	// check the nginx master process is running
+ 	fs, err := proc.NewFS("/proc", false)
+ 	if err != nil {

modules/402-ingress-nginx/images/controller-0-49/patches/lua-info.patch

@@ -0,0 +1,157 @@
+diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go
+index ff8557004..6802e006d 100644
+--- a/internal/ingress/controller/controller.go
++++ b/internal/ingress/controller/controller.go
+@@ -511,7 +511,7 @@ func (n *NGINXController) getBackendServers(ingresses []*ingress.Ingress) ([]*in
+ 			}
+
+ 			for _, path := range rule.HTTP.Paths {
+-				upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
++				upsName := upstreamName(ing.Namespace, ing.Name, path.Backend.ServiceName, path.Backend.ServicePort)
+
+ 				ups := upstreams[upsName]
+
+@@ -706,11 +706,12 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B
+
+ 		var defBackend string
+ 		if ing.Spec.Backend != nil {
+-			defBackend = upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
++			defBackend = upstreamName(ing.Namespace, ing.Name, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
+
+ 			klog.V(3).Infof("Creating upstream %q", defBackend)
+ 			upstreams[defBackend] = newUpstream(defBackend)
+
++			upstreams[defBackend].Ingress = &ing.Ingress
+ 			upstreams[defBackend].UpstreamHashBy.UpstreamHashBy = anns.UpstreamHashBy.UpstreamHashBy
+ 			upstreams[defBackend].UpstreamHashBy.UpstreamHashBySubset = anns.UpstreamHashBy.UpstreamHashBySubset
+ 			upstreams[defBackend].UpstreamHashBy.UpstreamHashBySubsetSize = anns.UpstreamHashBy.UpstreamHashBySubsetSize
+@@ -764,7 +765,7 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B
+ 			}
+
+ 			for _, path := range rule.HTTP.Paths {
+-				name := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
++				name := upstreamName(ing.Namespace, ing.Name, path.Backend.ServiceName, path.Backend.ServicePort)
+
+ 				if _, ok := upstreams[name]; ok {
+ 					continue
+@@ -772,6 +773,7 @@ func (n *NGINXController) createUpstreams(data []*ingress.Ingress, du *ingress.B
+
+ 				klog.V(3).Infof("Creating upstream %q", name)
+ 				upstreams[name] = newUpstream(name)
++				upstreams[name].Ingress = &ing.Ingress
+ 				upstreams[name].Port = path.Backend.ServicePort
+
+ 				upstreams[name].UpstreamHashBy.UpstreamHashBy = anns.UpstreamHashBy.UpstreamHashBy
+@@ -983,7 +985,7 @@ func (n *NGINXController) createServers(data []*ingress.Ingress,
+ 		}
+
+ 		if ing.Spec.Backend != nil {
+-			defUpstream := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
++			defUpstream := upstreamName(ing.Namespace, ing.Name, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
+
+ 			if backendUpstream, ok := upstreams[defUpstream]; ok {
+ 				// use backend specified in Ingress as the default backend for all its rules
+@@ -1235,7 +1237,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres
+
+ 	// merge catch-all alternative backends
+ 	if ing.Spec.Backend != nil {
+-		upsName := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
++		upsName := upstreamName(ing.Namespace, ing.Name, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort)
+
+ 		altUps := upstreams[upsName]
+
+@@ -1272,7 +1274,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres
+
+ 	for _, rule := range ing.Spec.Rules {
+ 		for _, path := range rule.HTTP.Paths {
+-			upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort)
++			upsName := upstreamName(ing.Namespace, ing.Name, path.Backend.ServiceName, path.Backend.ServicePort)
+
+ 			altUps := upstreams[upsName]
+
+diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go
+index 0cc1bbe34..4416028e1 100644
+--- a/internal/ingress/controller/nginx.go
++++ b/internal/ingress/controller/nginx.go
+@@ -35,6 +35,8 @@ import (
+ 	"text/template"
+ 	"time"
+
++	"k8s.io/api/networking/v1beta1"
++	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ 	proxyproto "github.com/armon/go-proxyproto"
+ 	"github.com/eapache/channels"
+ 	apiv1 "k8s.io/api/core/v1"
+@@ -960,8 +962,12 @@ func configureBackends(rawBackends []*ingress.Backend) error {
+
+ 	for i, backend := range rawBackends {
+ 		var service *apiv1.Service
++		var v1beta1ingress *v1beta1.Ingress
+ 		if backend.Service != nil {
+-			service = &apiv1.Service{Spec: backend.Service.Spec}
++			service = &apiv1.Service{ObjectMeta: metav1.ObjectMeta{Name: backend.Service.ObjectMeta.Name}, Spec: backend.Service.Spec}
++		}
++		if backend.Ingress != nil {
++			v1beta1ingress = &v1beta1.Ingress{ObjectMeta: metav1.ObjectMeta{Name: backend.Ingress.ObjectMeta.Name}}
+ 		}
+ 		luaBackend := &ingress.Backend{
+ 			Name:                 backend.Name,
+@@ -971,6 +977,7 @@ func configureBackends(rawBackends []*ingress.Backend) error {
+ 			UpstreamHashBy:       backend.UpstreamHashBy,
+ 			LoadBalancing:        backend.LoadBalancing,
+ 			Service:              service,
++			Ingress:              v1beta1ingress,
+ 			NoServer:             backend.NoServer,
+ 			TrafficShapingPolicy: backend.TrafficShapingPolicy,
+ 			AlternativeBackends:  backend.AlternativeBackends,
+diff --git a/internal/ingress/controller/util.go b/internal/ingress/controller/util.go
+index 56273f639..80aa82be5 100644
+--- a/internal/ingress/controller/util.go
++++ b/internal/ingress/controller/util.go
+@@ -43,8 +43,8 @@ func newUpstream(name string) *ingress.Backend {
+ }
+
+ // upstreamName returns a formatted upstream name based on namespace, service, and port
+-func upstreamName(namespace string, service string, port intstr.IntOrString) string {
+-	return fmt.Sprintf("%v-%v-%v", namespace, service, port.String())
++func upstreamName(namespace string, ingress string, service string, port intstr.IntOrString) string {
++	return fmt.Sprintf("%v-%v-%v-%v", namespace, ingress, service, port.String())
+ }
+
+ // sysctlSomaxconn returns the maximum number of connections that can be queued
+diff --git a/internal/ingress/types.go b/internal/ingress/types.go
+index 63714d739..97a0a3984 100644
+--- a/internal/ingress/types.go
++++ b/internal/ingress/types.go
+@@ -84,6 +84,7 @@ type Backend struct {
+ 	// Name represents an unique apiv1.Service name formatted as <namespace>-<name>-<port>
+ 	Name    string             `json:"name"`
+ 	Service *apiv1.Service     `json:"service,omitempty"`
++	Ingress *networking.Ingress `json:"ingress,omitempty"`
+ 	Port    intstr.IntOrString `json:"port"`
+ 	// SSLPassthrough indicates that Ingress controller will delegate TLS termination to the endpoints.
+ 	SSLPassthrough bool `json:"sslPassthrough"`
+diff --git a/internal/ingress/zz_generated.deepcopy.go b/internal/ingress/zz_generated.deepcopy.go
+index e8dfd1903..5eecbfa6f 100644
+--- a/internal/ingress/zz_generated.deepcopy.go
++++ b/internal/ingress/zz_generated.deepcopy.go
+@@ -22,6 +22,7 @@ package ingress
+
+ import (
+ 	v1 "k8s.io/api/core/v1"
++	v1beta1 "k8s.io/api/networking/v1beta1"
+ )
+
+ // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+@@ -32,6 +33,11 @@ func (in *Backend) DeepCopyInto(out *Backend) {
+ 		*out = new(v1.Service)
+ 		(*in).DeepCopyInto(*out)
+ 	}
++	if in.Ingress != nil {
++		in, out := &in.Ingress, &out.Ingress
++		*out = new(v1beta1.Ingress)
++		(*in).DeepCopyInto(*out)
++	}
+ 	out.Port = in.Port
+ 	if in.Endpoints != nil {
+ 		in, out := &in.Endpoints, &out.Endpoints