From 03928b04ecec19746cb923778e0ff8b37f0407e4 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:45:45 +0100 Subject: [PATCH 01/55] accept otel configmap keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/config_params.go | 8 +++ internal/configs/configmaps.go | 73 +++++++++++++++++++++++ internal/configs/version1/config.go | 8 +++ internal/configs/version1/nginx-plus.tmpl | 19 ++++++ 4 files changed, 108 insertions(+) diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index ec8d0c8a62..f4e0e99ec4 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -34,6 +34,14 @@ type ConfigParams struct { MainLogFormat []string MainLogFormatEscaping string MainMainSnippets []string + MainOtelEnabled bool + MainOtelLoadModule bool + MainOtelGlobalTraceEnabled bool + MainOtelExporterEndpoint string + MainOtelExporterTrustedCA string + MainOtelExporterHeaderName string + MainOtelExporterHeaderValue string + MainOtelServiceName string MainServerNamesHashBucketSize string MainServerNamesHashMaxSize string MainStreamLogFormat []string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 91a010c752..4cab3892ac 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -530,6 +530,71 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has } } + if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { + otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) + if otelExporterEndpoint != "" { + cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint + } + } + + if otelExporterTrustedCA, exists := cfgm.Data["otel-exporter-trusted-ca"]; exists { + otelExporterTrustedCA = strings.TrimSpace(otelExporterTrustedCA) + if otelExporterTrustedCA != "" { + cfgParams.MainOtelExporterTrustedCA = otelExporterTrustedCA + } + } + + if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { + otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) + if otelExporterHeaderName != "" { + cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName + } + } + + if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists { + otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue) + if otelExporterHeaderValue != "" { + cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue + } + } + + if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists { + otelServiceName = strings.TrimSpace(otelServiceName) + if otelServiceName != "" { + cfgParams.MainOtelServiceName = otelServiceName + } + } + + if otelGlobalTraceEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-global-trace-enabled", cfgm); exists { + if err != nil { + nl.Error(l, err) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) + configOk = false + } + cfgParams.MainOtelGlobalTraceEnabled = otelGlobalTraceEnabled + } + + if cfgParams.MainOtelExporterEndpoint != "" { + cfgParams.MainOtelLoadModule = true + } + + if otelEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-enabled", cfgm); exists { + if err != nil { + nl.Error(l, err) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) + configOk = false + } else { + if cfgParams.MainOtelLoadModule { + cfgParams.MainOtelEnabled = otelEnabled + } else { + errorText := fmt.Sprintf("ConfigMap %s/%s: 'otel-enabled' is ignored because 'otel-exporter-endpoint' is not set, ignoring", cfgm.GetNamespace(), cfgm.GetName()) + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + configOk = false + } + } + } + if hasAppProtect { if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists { if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" { @@ -913,6 +978,14 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config NginxStatus: staticCfgParams.NginxStatus, NginxStatusAllowCIDRs: staticCfgParams.NginxStatusAllowCIDRs, NginxStatusPort: staticCfgParams.NginxStatusPort, + MainOtelEnabled: config.MainOtelEnabled, + MainOtelLoadModule: config.MainOtelLoadModule, + MainOtelGlobalTraceEnabled: config.MainOtelGlobalTraceEnabled, + MainOtelExporterEndpoint: config.MainOtelExporterEndpoint, + MainOtelExporterTrustedCA: config.MainOtelExporterTrustedCA, + MainOtelExporterHeaderName: config.MainOtelExporterHeaderName, + MainOtelExporterHeaderValue: config.MainOtelExporterHeaderValue, + MainOtelServiceName: config.MainOtelServiceName, ProxyProtocol: config.ProxyProtocol, ResolverAddresses: config.ResolverAddresses, ResolverIPV6: config.ResolverIPV6, diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 8b510e315f..6ac8647ff9 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -240,6 +240,14 @@ type MainConfig struct { NginxStatus bool NginxStatusAllowCIDRs []string NginxStatusPort int + MainOtelEnabled bool + MainOtelLoadModule bool + MainOtelGlobalTraceEnabled bool + MainOtelExporterEndpoint string + MainOtelExporterTrustedCA string + MainOtelExporterHeaderName string + MainOtelExporterHeaderValue string + MainOtelServiceName string ProxyProtocol bool ResolverAddresses []string ResolverIPV6 bool diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9b5a0738c5..8f9f2552f4 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -12,6 +12,9 @@ daemon off; error_log stderr {{.ErrorLogLevel}}; pid /var/lib/nginx/nginx.pid; +{{- if .MainOtelLoadModule}} +#load_module modules/ngx_otel_module.so; +{{- end}} {{- if .AppProtectLoadModule}} load_module modules/ngx_http_app_protect_module.so; {{- end}} @@ -142,6 +145,22 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} + + {{- if .MainOtelEnabled}} + # otel_exporter { + # endpoint {{ .MainOtelExporterEndpoint}}; + # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + {{ if .MainOtelExporterTrustedCA}} + # trusted_certificate ; + {{- end }} + {{ if .MainOtelServiceName}} + # otel_service_name {{ .MainOtelServiceName }}; + # } + {{- end }} + {{ if .MainOtelGlobalTraceEnabled }} + # otel_trace on; + {{- end}} + {{- end}} {{ $resolverIPV6HTTPBool := boolToPointerBool .ResolverIPV6 -}} {{ makeResolver .ResolverAddresses .ResolverValid $resolverIPV6HTTPBool }} From cc7808e60fba5e9b6fe11705d7685845d935be41 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 11 Apr 2025 14:45:29 +0100 Subject: [PATCH 02/55] add otel-module to NIC images Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 09ff7093c4..738fed269c 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -21,15 +21,26 @@ FROM golang:1.24-alpine@sha256:ef18ee7117463ac1055f5a370ed18b8750f01589f13ea0b48 ############################################# Base image for Alpine ############################################# FROM nginx:1.27.4-alpine@sha256:4ff102c5d78d254a6f0da062b3cf39eaf07f01eec0927fd21e219d0af8bc0591 AS alpine -RUN apk add --no-cache libcap libstdc++ +RUN apk add --no-cache libcap libstdc++ \ + && printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ + && apk add --no-cache libcap libstdc++ nginx-module-otel \ + && sed -i -e '/nginx.org/d' /etc/apk/repositories ############################################# Base image for Debian ############################################# FROM nginx:1.27.4@sha256:09369da6b10306312cd908661320086bf87fbae1b6b0c49a1f50ba531fef2eab AS debian RUN apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y libcap2-bin - + && apt-get install --no-install-recommends --no-install-suggests -y \ + libcap2-bin curl gnupg2 ca-certificates lsb-release debian-archive-keyring \ + && curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \ + http://nginx.org/packages/mainline/debian `lsb_release -cs` nginx" > /etc/apt/sources.list.d/nginx.list \ + && printf "%s" "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" > /etc/apt/preferences.d/99nginx \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y nginx-module-otel \ + && apt-get purge --auto-remove -y gnupg2 lsb-release curl \ + && rm -rf /var/lib/apt/lists/* /etc/apt/preferences.d/99nginx /etc/apt/sources.list.d/nginx.list ############################################# NGINX files ############################################# FROM scratch AS nginx-files @@ -109,7 +120,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ export $(cat /tmp/user_agent) \ && printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check libcap libcurl \ + && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check libcap libcurl \ && mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ldconfig /usr/local/lib/ \ && sed -i -e '/nginx.com/d' /etc/apk/repositories @@ -151,7 +162,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://pkgs.nginx.com/app-protect-security-updates/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -187,7 +198,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect-x-plus/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -226,7 +237,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && gpg --dearmor -o /usr/share/keyrings/app-protect-archive-keyring.gpg /tmp/app-protect-security-updates.key \ && cp /tmp/nginx-plus.sources /etc/apt/sources.list.d/nginx-plus.sources \ && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && apt-get purge --auto-remove -y gpg \ && mkdir -p /etc/nginx/reporting/ \ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ @@ -346,7 +357,7 @@ RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_s printf "%s\n" "[nginx]" "name=nginx repo" \ "baseurl=https://nginx.org/packages/mainline/centos/9/\$basearch/" \ "gpgcheck=1" "enabled=1" "module_hotfixes=true" > /etc/yum.repos.d/nginx.repo \ - && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-image-filter nginx-module-xslt \ + && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel nginx-module-image-filter nginx-module-xslt \ && rm /etc/yum.repos.d/nginx.repo; \ fi \ && ubi-clean.sh @@ -368,7 +379,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ubi-setup.sh \ - && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && ubi-clean.sh @@ -473,7 +484,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ && subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \ @@ -520,7 +531,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ ## end of duplicated code && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ From fe36cac604b36e8f9d5bcc1b96b48786281546ea Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:15:53 +0100 Subject: [PATCH 03/55] remove unused config map keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 17 ----------------- internal/configs/version1/nginx-plus.tmpl | 21 ++++++++++++--------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 4cab3892ac..d509769118 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -578,23 +578,6 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has cfgParams.MainOtelLoadModule = true } - if otelEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-enabled", cfgm); exists { - if err != nil { - nl.Error(l, err) - eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) - configOk = false - } else { - if cfgParams.MainOtelLoadModule { - cfgParams.MainOtelEnabled = otelEnabled - } else { - errorText := fmt.Sprintf("ConfigMap %s/%s: 'otel-enabled' is ignored because 'otel-exporter-endpoint' is not set, ignoring", cfgm.GetNamespace(), cfgm.GetName()) - nl.Error(l, errorText) - eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) - configOk = false - } - } - } - if hasAppProtect { if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists { if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" { diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 8f9f2552f4..c637614df0 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -13,7 +13,7 @@ error_log stderr {{.ErrorLogLevel}}; pid /var/lib/nginx/nginx.pid; {{- if .MainOtelLoadModule}} -#load_module modules/ngx_otel_module.so; +load_module modules/ngx_otel_module.so; {{- end}} {{- if .AppProtectLoadModule}} load_module modules/ngx_http_app_protect_module.so; @@ -145,20 +145,23 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} - - {{- if .MainOtelEnabled}} - # otel_exporter { - # endpoint {{ .MainOtelExporterEndpoint}}; - # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + + {{- if .MainOtelLoadModule}} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint}}; + {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} {{ if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} + } + {{ if .MainOtelServiceName}} - # otel_service_name {{ .MainOtelServiceName }}; - # } + otel_service_name {{ .MainOtelServiceName }}; {{- end }} {{ if .MainOtelGlobalTraceEnabled }} - # otel_trace on; + otel_trace on; {{- end}} {{- end}} From 639bf3fe57e6e2a8242e2028f2cdfe9319411f5f Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 14 Apr 2025 13:02:40 +0100 Subject: [PATCH 04/55] update templates to add OSS support --- internal/configs/version1/nginx-plus.tmpl | 8 ++++---- internal/configs/version1/nginx.tmpl | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index c637614df0..eefd0a98e5 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -149,18 +149,18 @@ http { {{- if .MainOtelLoadModule}} otel_exporter { endpoint {{ .MainOtelExporterEndpoint}}; - {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{ if .MainOtelExporterTrustedCA}} + {{- if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} } - {{ if .MainOtelServiceName}} + {{- if .MainOtelServiceName}} otel_service_name {{ .MainOtelServiceName }}; {{- end }} - {{ if .MainOtelGlobalTraceEnabled }} + {{- if .MainOtelGlobalTraceEnabled }} otel_trace on; {{- end}} {{- end}} diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index 5b290b381d..b54acefa66 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -11,6 +11,10 @@ daemon off; error_log stderr {{.ErrorLogLevel}}; pid /var/lib/nginx/nginx.pid; +{{- if .MainOtelLoadModule}} +load_module modules/ngx_otel_module.so; +{{- end}} + {{- if .MainSnippets}} {{range $value := .MainSnippets}} {{$value}}{{end}} @@ -104,6 +108,25 @@ http { ssl_dhparam {{.SSLDHParam}}; {{- end}} + {{- if .MainOtelLoadModule}} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint}}; + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} + {{- if .MainOtelExporterTrustedCA}} + # trusted_certificate ; + {{- end }} + } + + {{- if .MainOtelServiceName}} + otel_service_name {{ .MainOtelServiceName }}; + {{- end }} + {{- if .MainOtelGlobalTraceEnabled }} + otel_trace on; + {{- end}} + {{- end}} + server { # required to support the Websocket protocol in VirtualServer/VirtualServerRoutes set $default_connection_header ""; From b66feb2b45e786634a9909b99f6ff08e97032646 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:45:45 +0100 Subject: [PATCH 05/55] accept otel configmap keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 65 +++++++++++++++++++++++ internal/configs/version1/nginx-plus.tmpl | 16 ++++++ 2 files changed, 81 insertions(+) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index d509769118..8d05b6b09a 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -578,6 +578,71 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has cfgParams.MainOtelLoadModule = true } + if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { + otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) + if otelExporterEndpoint != "" { + cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint + } + } + + if otelExporterTrustedCA, exists := cfgm.Data["otel-exporter-trusted-ca"]; exists { + otelExporterTrustedCA = strings.TrimSpace(otelExporterTrustedCA) + if otelExporterTrustedCA != "" { + cfgParams.MainOtelExporterTrustedCA = otelExporterTrustedCA + } + } + + if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { + otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) + if otelExporterHeaderName != "" { + cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName + } + } + + if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists { + otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue) + if otelExporterHeaderValue != "" { + cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue + } + } + + if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists { + otelServiceName = strings.TrimSpace(otelServiceName) + if otelServiceName != "" { + cfgParams.MainOtelServiceName = otelServiceName + } + } + + if otelGlobalTraceEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-global-trace-enabled", cfgm); exists { + if err != nil { + nl.Error(l, err) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) + configOk = false + } + cfgParams.MainOtelGlobalTraceEnabled = otelGlobalTraceEnabled + } + + if cfgParams.MainOtelExporterEndpoint != "" { + cfgParams.MainOtelLoadModule = true + } + + if otelEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-enabled", cfgm); exists { + if err != nil { + nl.Error(l, err) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) + configOk = false + } else { + if cfgParams.MainOtelLoadModule { + cfgParams.MainOtelEnabled = otelEnabled + } else { + errorText := fmt.Sprintf("ConfigMap %s/%s: 'otel-enabled' is ignored because 'otel-exporter-endpoint' is not set, ignoring", cfgm.GetNamespace(), cfgm.GetName()) + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + configOk = false + } + } + } + if hasAppProtect { if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists { if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" { diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index eefd0a98e5..cc0be3e5c9 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -145,6 +145,22 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} + + {{- if .MainOtelEnabled}} + # otel_exporter { + # endpoint {{ .MainOtelExporterEndpoint}}; + # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + {{ if .MainOtelExporterTrustedCA}} + # trusted_certificate ; + {{- end }} + {{ if .MainOtelServiceName}} + # otel_service_name {{ .MainOtelServiceName }}; + # } + {{- end }} + {{ if .MainOtelGlobalTraceEnabled }} + # otel_trace on; + {{- end}} + {{- end}} {{- if .MainOtelLoadModule}} otel_exporter { From 27114b1eb710456af7a5c43e8e9a9a7e3dd6241f Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 11 Apr 2025 14:45:29 +0100 Subject: [PATCH 06/55] add otel-module to NIC images Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 738fed269c..63b387251f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -21,14 +21,13 @@ FROM golang:1.24-alpine@sha256:ef18ee7117463ac1055f5a370ed18b8750f01589f13ea0b48 ############################################# Base image for Alpine ############################################# FROM nginx:1.27.4-alpine@sha256:4ff102c5d78d254a6f0da062b3cf39eaf07f01eec0927fd21e219d0af8bc0591 AS alpine -RUN apk add --no-cache libcap libstdc++ \ - && printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ +RUN printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ && apk add --no-cache libcap libstdc++ nginx-module-otel \ && sed -i -e '/nginx.org/d' /etc/apk/repositories ############################################# Base image for Debian ############################################# -FROM nginx:1.27.4@sha256:09369da6b10306312cd908661320086bf87fbae1b6b0c49a1f50ba531fef2eab AS debian +FROM nginx:1.27.4@sha256:124b44bfc9ccd1f3cedf4b592d4d1e8bddb78b51ec2ed5056c52d3692baebc19 AS debian RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ From 1c1542c9ed814eb760b14adbe206eba8bdf5947c Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:15:53 +0100 Subject: [PATCH 07/55] remove unused config map keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 17 ----------------- internal/configs/version1/nginx-plus.tmpl | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 25 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 8d05b6b09a..f193c00123 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -626,23 +626,6 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has cfgParams.MainOtelLoadModule = true } - if otelEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-enabled", cfgm); exists { - if err != nil { - nl.Error(l, err) - eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) - configOk = false - } else { - if cfgParams.MainOtelLoadModule { - cfgParams.MainOtelEnabled = otelEnabled - } else { - errorText := fmt.Sprintf("ConfigMap %s/%s: 'otel-enabled' is ignored because 'otel-exporter-endpoint' is not set, ignoring", cfgm.GetNamespace(), cfgm.GetName()) - nl.Error(l, errorText) - eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) - configOk = false - } - } - } - if hasAppProtect { if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists { if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" { diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index cc0be3e5c9..ca779514f6 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -145,20 +145,23 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} - - {{- if .MainOtelEnabled}} - # otel_exporter { - # endpoint {{ .MainOtelExporterEndpoint}}; - # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + + {{- if .MainOtelLoadModule}} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint}}; + {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} {{ if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} + } + {{ if .MainOtelServiceName}} - # otel_service_name {{ .MainOtelServiceName }}; - # } + otel_service_name {{ .MainOtelServiceName }}; {{- end }} {{ if .MainOtelGlobalTraceEnabled }} - # otel_trace on; + otel_trace on; {{- end}} {{- end}} From e163eb61b41e52a58da9a33c6beb9cad0fa572fc Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 29 Apr 2025 13:28:57 +0100 Subject: [PATCH 08/55] remove unused otel flag --- internal/configs/config_params.go | 1 - internal/configs/configmaps.go | 1 - 2 files changed, 2 deletions(-) diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index f4e0e99ec4..5c904ef5e3 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -34,7 +34,6 @@ type ConfigParams struct { MainLogFormat []string MainLogFormatEscaping string MainMainSnippets []string - MainOtelEnabled bool MainOtelLoadModule bool MainOtelGlobalTraceEnabled bool MainOtelExporterEndpoint string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index f193c00123..fe37f58de1 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -1009,7 +1009,6 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config NginxStatus: staticCfgParams.NginxStatus, NginxStatusAllowCIDRs: staticCfgParams.NginxStatusAllowCIDRs, NginxStatusPort: staticCfgParams.NginxStatusPort, - MainOtelEnabled: config.MainOtelEnabled, MainOtelLoadModule: config.MainOtelLoadModule, MainOtelGlobalTraceEnabled: config.MainOtelGlobalTraceEnabled, MainOtelExporterEndpoint: config.MainOtelExporterEndpoint, From d4745e8f10977442277ec7c70be5db0ec55e1232 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 29 Apr 2025 13:51:51 +0100 Subject: [PATCH 09/55] rename mgmt trusted secret function --- cmd/nginx-ingress/main.go | 4 ++-- internal/configs/version1/config.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 30f4cfa0cf..4678e1ea97 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -166,7 +166,7 @@ func main() { logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) } - if err := processTrustedCertSecret(kubeClient, nginxManager, mgmtCfgParams, controllerNamespace); err != nil { + if err := processMgmtTrustedCertSecret(kubeClient, nginxManager, mgmtCfgParams, controllerNamespace); err != nil { logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) } @@ -371,7 +371,7 @@ func processClientAuthSecret(kubeClient *kubernetes.Clientset, nginxManager ngin return nil } -func processTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, mgmtCfgParams *configs.MGMTConfigParams, controllerNamespace string) error { +func processMgmtTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, mgmtCfgParams *configs.MGMTConfigParams, controllerNamespace string) error { if mgmtCfgParams.Secrets.TrustedCert == "" { return nil } diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 6ac8647ff9..a42730058a 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -240,7 +240,6 @@ type MainConfig struct { NginxStatus bool NginxStatusAllowCIDRs []string NginxStatusPort int - MainOtelEnabled bool MainOtelLoadModule bool MainOtelGlobalTraceEnabled bool MainOtelExporterEndpoint string From b277d827a88358aad4d713b7215b6ffe9cba78d8 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 29 Apr 2025 15:27:39 +0100 Subject: [PATCH 10/55] add CA to Otel exporter on startup --- cmd/nginx-ingress/main.go | 22 ++++++++++++++++++++++ internal/configs/configmaps.go | 2 +- internal/configs/version1/nginx-plus.tmpl | 10 +++++----- internal/configs/version1/nginx.tmpl | 8 ++++---- 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 4678e1ea97..262647ffc5 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -189,6 +189,7 @@ func main() { if err != nil { logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) } + globalConfigurationValidator := createGlobalConfigurationValidator() mustProcessGlobalConfiguration(ctx) @@ -196,6 +197,10 @@ func main() { cfgParams := configs.NewDefaultConfigParams(ctx, *nginxPlus) cfgParams = processConfigMaps(kubeClient, cfgParams, nginxManager, templateExecutor, eventRecorder) + if err := processOtelTrustedCertSecret(kubeClient, nginxManager, cfgParams, controllerNamespace); err != nil { + logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) + } + staticCfgParams := &configs.StaticConfigParams{ DisableIPV6: *disableIPV6, DefaultHTTPListenerPort: *defaultHTTPListenerPort, @@ -392,6 +397,23 @@ func processMgmtTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager return nil } +func processOtelTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, cfgParams *configs.ConfigParams, controllerNamespace string) error { + if cfgParams.MainOtelExporterTrustedCA == "" { + return nil + } + + trustedCertSecretNsName := controllerNamespace + "/" + cfgParams.MainOtelExporterTrustedCA + + secret, err := getAndValidateSecret(kubeClient, trustedCertSecretNsName, secrets.SecretTypeCA) + if err != nil { + return fmt.Errorf("error trying to get the trusted cert secret %v: %w", trustedCertSecretNsName, err) + } + + caBytes, _ := configs.GenerateCAFileContent(secret) + nginxManager.CreateSecret(fmt.Sprintf("%s-%s-%s", controllerNamespace, cfgParams.MainOtelExporterTrustedCA, configs.CACrtKey), caBytes, nginx.ReadWriteOnlyFileMode) + return nil +} + func mustCreateConfigAndKubeClient(ctx context.Context) (*rest.Config, *kubernetes.Clientset) { l := nl.LoggerFromContext(ctx) var config *rest.Config diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index fe37f58de1..4554e828d7 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -1012,7 +1012,7 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config MainOtelLoadModule: config.MainOtelLoadModule, MainOtelGlobalTraceEnabled: config.MainOtelGlobalTraceEnabled, MainOtelExporterEndpoint: config.MainOtelExporterEndpoint, - MainOtelExporterTrustedCA: config.MainOtelExporterTrustedCA, + MainOtelExporterTrustedCA: fmt.Sprintf("%s-%s-%s", os.Getenv("POD_NAMESPACE"), config.MainOtelExporterTrustedCA, CACrtKey), MainOtelExporterHeaderName: config.MainOtelExporterHeaderName, MainOtelExporterHeaderValue: config.MainOtelExporterHeaderValue, MainOtelServiceName: config.MainOtelServiceName, diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index ca779514f6..40f414373b 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -146,14 +146,14 @@ http { ssl_dhparam {{.SSLDHParam}}; {{- end}} - {{- if .MainOtelLoadModule}} + {{- if .MainOtelLoadModule }} otel_exporter { - endpoint {{ .MainOtelExporterEndpoint}}; - {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + endpoint {{ .MainOtelExporterEndpoint }}; + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{ if .MainOtelExporterTrustedCA}} - # trusted_certificate ; + {{- if .MainOtelExporterTrustedCA }} + trusted_certificate /etc/nginx/secrets/{{ .MainOtelExporterTrustedCA }}; {{- end }} } diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index b54acefa66..2d1822c8ee 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -108,14 +108,14 @@ http { ssl_dhparam {{.SSLDHParam}}; {{- end}} - {{- if .MainOtelLoadModule}} + {{- if .MainOtelLoadModule }} otel_exporter { - endpoint {{ .MainOtelExporterEndpoint}}; + endpoint {{ .MainOtelExporterEndpoint }}; {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{- if .MainOtelExporterTrustedCA}} - # trusted_certificate ; + {{- if .MainOtelExporterTrustedCA }} + trusted_certificate /etc/nginx/secrets/{{ .MainOtelExporterTrustedCA }}; {{- end }} } From 57804209e285d84c1b243cb4f1b9ceeacf5d804d Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 29 Apr 2025 17:47:18 +0100 Subject: [PATCH 11/55] add runtime config for Otel CA cert updates --- internal/configs/configmaps.go | 7 +++- internal/configs/configurator.go | 5 ++- internal/k8s/controller.go | 61 +++++++++++++++++++++++++------- 3 files changed, 58 insertions(+), 15 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 4554e828d7..cd63bd4767 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -988,6 +988,11 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config ResolverValid: config.ZoneSync.ResolverValid, } + mainOtelExporterTrustedCA := "" + if config.MainOtelExporterTrustedCA != "" { + mainOtelExporterTrustedCA = fmt.Sprintf("%s-%s-%s", os.Getenv("POD_NAMESPACE"), config.MainOtelExporterTrustedCA, CACrtKey) + } + nginxCfg := &version1.MainConfig{ AccessLog: config.MainAccessLog, DefaultServerAccessLogOff: config.DefaultServerAccessLogOff, @@ -1012,7 +1017,7 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config MainOtelLoadModule: config.MainOtelLoadModule, MainOtelGlobalTraceEnabled: config.MainOtelGlobalTraceEnabled, MainOtelExporterEndpoint: config.MainOtelExporterEndpoint, - MainOtelExporterTrustedCA: fmt.Sprintf("%s-%s-%s", os.Getenv("POD_NAMESPACE"), config.MainOtelExporterTrustedCA, CACrtKey), + MainOtelExporterTrustedCA: mainOtelExporterTrustedCA, MainOtelExporterHeaderName: config.MainOtelExporterHeaderName, MainOtelExporterHeaderValue: config.MainOtelExporterHeaderValue, MainOtelServiceName: config.MainOtelServiceName, diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index 110ab16d6f..ba370848e8 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -826,8 +826,11 @@ func generateTLSPassthroughHostsConfig(tlsPassthroughPairs map[string]tlsPassthr // AddOrUpdateCASecret writes the secret content to disk returning the files added/updated func (cnf *Configurator) AddOrUpdateCASecret(secret *api_v1.Secret, crtFileName, crlFileName string) string { crtData, crlData := GenerateCAFileContent(secret) + crlFilePath := "" crtFilePath := cnf.nginxManager.CreateSecret(crtFileName, crtData, nginx.ReadWriteOnlyFileMode) - crlFilePath := cnf.nginxManager.CreateSecret(crlFileName, crlData, nginx.ReadWriteOnlyFileMode) + if len(crlData) > 0 { + crlFilePath = cnf.nginxManager.CreateSecret(crlFileName, crlData, nginx.ReadWriteOnlyFileMode) + } return fmt.Sprintf("%s %s", crtFilePath, crlFilePath) } diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 3435e07de1..108931f050 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -102,11 +102,12 @@ type podEndpoint struct { } type specialSecrets struct { - defaultServerSecret string - wildcardTLSSecret string - licenseSecret string - clientAuthSecret string - trustedCertSecret string + defaultServerSecret string + wildcardTLSSecret string + licenseSecret string + clientAuthSecret string + mgmtTrustedCertSecret string + otelTrustedCertSecret string } type controllerMetadata struct { @@ -247,14 +248,19 @@ type NewLoadBalancerControllerInput struct { // NewLoadBalancerController creates a controller func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalancerController { + otelTrustedCertSecret := "" + if input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA != "" { + otelTrustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA) + } specialSecrets := specialSecrets{ - defaultServerSecret: input.DefaultServerSecret, - wildcardTLSSecret: input.WildcardTLSSecret, + defaultServerSecret: input.DefaultServerSecret, + wildcardTLSSecret: input.WildcardTLSSecret, + otelTrustedCertSecret: otelTrustedCertSecret, } if input.IsNginxPlus { specialSecrets.licenseSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.License) specialSecrets.clientAuthSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.ClientAuth) - specialSecrets.trustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.TrustedCert) + specialSecrets.mgmtTrustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.TrustedCert) } lbc := &LoadBalancerController{ client: input.KubeClient, @@ -924,7 +930,7 @@ func (lbc *LoadBalancerController) updateAllConfigs() { if _, hasCRL := secret.Data[configs.CACrlKey]; hasCRL { lbc.configurator.MgmtCfgParams.Secrets.TrustedCRL = secret.Name } - lbc.specialSecrets.trustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name) + lbc.specialSecrets.mgmtTrustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name) lbc.handleSpecialSecretUpdate(secret, reloadNginx) } // update special ClientAuth secret in mgmtConfigParams @@ -937,6 +943,15 @@ func (lbc *LoadBalancerController) updateAllConfigs() { lbc.handleSpecialSecretUpdate(secret, reloadNginx) } } + // update special Otel CA secret in configParams + if cfgParams.MainOtelExporterTrustedCA != "" { + secret, err := lbc.client.CoreV1().Secrets(lbc.configMap.GetNamespace()).Get(context.TODO(), cfgParams.MainOtelExporterTrustedCA, meta_v1.GetOptions{}) + if err != nil { + nl.Errorf(lbc.Logger, "secret %s/%s: %v", lbc.configMap.GetNamespace(), cfgParams.MainOtelExporterTrustedCA, err) + } + lbc.specialSecrets.otelTrustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name) + lbc.handleSpecialSecretUpdate(secret, reloadNginx) + } resources := lbc.configuration.GetResources() nl.Debugf(lbc.Logger, "Updating %v resources", len(resources)) resourceExes := lbc.createExtendedResources(resources) @@ -1843,7 +1858,9 @@ func (lbc *LoadBalancerController) isSpecialSecret(secretName string) bool { return true case lbc.specialSecrets.clientAuthSecret: return true - case lbc.specialSecrets.trustedCertSecret: + case lbc.specialSecrets.mgmtTrustedCertSecret: + return true + case lbc.specialSecrets.otelTrustedCertSecret: return true default: return false @@ -1918,7 +1935,12 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr if ok := lbc.performNGINXReload(secret); !ok { return } - case lbc.specialSecrets.trustedCertSecret: + case lbc.specialSecrets.mgmtTrustedCertSecret: + lbc.updateAllConfigs() + if ok := lbc.performNGINXReload(secret); !ok { + return + } + case lbc.specialSecrets.otelTrustedCertSecret: lbc.updateAllConfigs() if ok := lbc.performNGINXReload(secret); !ok { return @@ -1944,7 +1966,12 @@ func (lbc *LoadBalancerController) writeSpecialSecrets(secret *api_v1.Secret, sp return false } case secrets.SecretTypeCA: - lbc.configurator.AddOrUpdateCASecret(secret, fmt.Sprintf("mgmt/%s", configs.CACrtKey), fmt.Sprintf("mgmt/%s", configs.CACrlKey)) + if lbc.specialSecrets.mgmtTrustedCertSecret != "" { + lbc.configurator.AddOrUpdateCASecret(secret, fmt.Sprintf("mgmt/%s", configs.CACrtKey), fmt.Sprintf("mgmt/%s", configs.CACrlKey)) + } + if lbc.specialSecrets.otelTrustedCertSecret != "" { + lbc.configurator.AddOrUpdateCASecret(secret, fmt.Sprintf("%s-%s-%s", lbc.metadata.namespace, lbc.configurator.CfgParams.MainOtelExporterTrustedCA, configs.CACrtKey), "") + } case api_v1.SecretTypeTLS: // if the secret name matches the specified if secretNsName == mgmtClientAuthNamespaceName { @@ -1981,7 +2008,7 @@ func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, return false } } - if secretNsName == lbc.specialSecrets.trustedCertSecret { + if secretNsName == lbc.specialSecrets.mgmtTrustedCertSecret { err := secrets.ValidateCASecret(secret) if err != nil { nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err) @@ -1997,6 +2024,14 @@ func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, return false } } + if secretNsName == lbc.specialSecrets.otelTrustedCertSecret { + err := secrets.ValidateCASecret(secret) + if err != nil { + nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err) + lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, nl.EventReasonRejected, "the special Secret %v was rejected, using the previous version: %v", secretNsName, err) + return false + } + } return true } From 359b874d0fec9145a78cfa700f178d463f5c676a Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 29 Apr 2025 20:12:31 +0100 Subject: [PATCH 12/55] add nil check --- internal/k8s/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 108931f050..151358649b 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -249,7 +249,7 @@ type NewLoadBalancerControllerInput struct { // NewLoadBalancerController creates a controller func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalancerController { otelTrustedCertSecret := "" - if input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA != "" { + if input.NginxConfigurator != nil && input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA != "" { otelTrustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA) } specialSecrets := specialSecrets{ From 626af16eb8ab37971bcbe9e86fd1eb895639e5c6 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:45:45 +0100 Subject: [PATCH 13/55] accept otel configmap keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/version1/nginx-plus.tmpl | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 40f414373b..e053b6fe59 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -145,6 +145,22 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} + + {{- if .MainOtelEnabled}} + # otel_exporter { + # endpoint {{ .MainOtelExporterEndpoint}}; + # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + {{ if .MainOtelExporterTrustedCA}} + # trusted_certificate ; + {{- end }} + {{ if .MainOtelServiceName}} + # otel_service_name {{ .MainOtelServiceName }}; + # } + {{- end }} + {{ if .MainOtelGlobalTraceEnabled }} + # otel_trace on; + {{- end}} + {{- end}} {{- if .MainOtelLoadModule }} otel_exporter { From 26435e534c7d2b55960ea4185a015d0e5cb81315 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 11 Apr 2025 14:45:29 +0100 Subject: [PATCH 14/55] add otel-module to NIC images Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 63b387251f..f493bbf89f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -457,7 +457,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode ############################################# Base image for UBI8 with NGINX Plus and App Protect WAF ############################################# -FROM redhat/ubi8@sha256:244e9858f9d8a2792a3dceb850b4fa8fdbd67babebfde42587bfa919d5d1ecef AS ubi-8-plus-nap +FROM redhat/ubi8@sha256:8bd1b6306f8164de7fb0974031a0f903bd3ab3e6bcab835854d3d9a1a74ea5db AS ubi-8-plus-nap ARG NAP_MODULES ARG NGINX_AGENT ARG NGINX_PLUS_VERSION From 30c6dff98d0eaca86a1e44926d73172f7695f4a8 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:15:53 +0100 Subject: [PATCH 15/55] remove unused config map keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/version1/nginx-plus.tmpl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index e053b6fe59..9faf04bed9 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -145,20 +145,23 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} - - {{- if .MainOtelEnabled}} - # otel_exporter { - # endpoint {{ .MainOtelExporterEndpoint}}; - # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + + {{- if .MainOtelLoadModule}} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint}}; + {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} {{ if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} + } + {{ if .MainOtelServiceName}} - # otel_service_name {{ .MainOtelServiceName }}; - # } + otel_service_name {{ .MainOtelServiceName }}; {{- end }} {{ if .MainOtelGlobalTraceEnabled }} - # otel_trace on; + otel_trace on; {{- end}} {{- end}} From a1e6939fc212f0ae40402790d38c595db02f7ce5 Mon Sep 17 00:00:00 2001 From: Paul Abel <128620221+pdabelf5@users.noreply.github.com> Date: Mon, 14 Apr 2025 11:32:36 +0100 Subject: [PATCH 16/55] Remove OpenTracing Support from NIC (#7633) --- build/Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index f493bbf89f..18e4ab7ddf 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -119,7 +119,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ export $(cat /tmp/user_agent) \ && printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check libcap libcurl \ + && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check libcap libcurl \ && mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ldconfig /usr/local/lib/ \ && sed -i -e '/nginx.com/d' /etc/apk/repositories @@ -161,7 +161,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://pkgs.nginx.com/app-protect-security-updates/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -197,7 +197,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect-x-plus/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -236,7 +236,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && gpg --dearmor -o /usr/share/keyrings/app-protect-archive-keyring.gpg /tmp/app-protect-security-updates.key \ && cp /tmp/nginx-plus.sources /etc/apt/sources.list.d/nginx-plus.sources \ && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ + && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ && apt-get purge --auto-remove -y gpg \ && mkdir -p /etc/nginx/reporting/ \ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ @@ -356,7 +356,7 @@ RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_s printf "%s\n" "[nginx]" "name=nginx repo" \ "baseurl=https://nginx.org/packages/mainline/centos/9/\$basearch/" \ "gpgcheck=1" "enabled=1" "module_hotfixes=true" > /etc/yum.repos.d/nginx.repo \ - && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel nginx-module-image-filter nginx-module-xslt \ + && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-image-filter nginx-module-xslt \ && rm /etc/yum.repos.d/nginx.repo; \ fi \ && ubi-clean.sh @@ -378,7 +378,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ubi-setup.sh \ - && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ + && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ && ubi-clean.sh @@ -483,7 +483,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ && subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \ @@ -530,7 +530,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ ## end of duplicated code && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ From 02f51e1f4b2c4337eb2107b885423f433900f4ed Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:45:45 +0100 Subject: [PATCH 17/55] accept otel configmap keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 37 +--------------------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index cd63bd4767..27c51c0abe 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -578,42 +578,7 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has cfgParams.MainOtelLoadModule = true } - if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { - otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) - if otelExporterEndpoint != "" { - cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint - } - } - - if otelExporterTrustedCA, exists := cfgm.Data["otel-exporter-trusted-ca"]; exists { - otelExporterTrustedCA = strings.TrimSpace(otelExporterTrustedCA) - if otelExporterTrustedCA != "" { - cfgParams.MainOtelExporterTrustedCA = otelExporterTrustedCA - } - } - - if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { - otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) - if otelExporterHeaderName != "" { - cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName - } - } - - if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists { - otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue) - if otelExporterHeaderValue != "" { - cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue - } - } - - if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists { - otelServiceName = strings.TrimSpace(otelServiceName) - if otelServiceName != "" { - cfgParams.MainOtelServiceName = otelServiceName - } - } - - if otelGlobalTraceEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-global-trace-enabled", cfgm); exists { + if otelEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-enabled", cfgm); exists { if err != nil { nl.Error(l, err) eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) From 8c01a284fc1277d034dda5f620599e5f8271c216 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 11 Apr 2025 14:45:29 +0100 Subject: [PATCH 18/55] add otel-module to NIC images Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 18e4ab7ddf..f493bbf89f 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -119,7 +119,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ export $(cat /tmp/user_agent) \ && printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check libcap libcurl \ + && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check libcap libcurl \ && mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ldconfig /usr/local/lib/ \ && sed -i -e '/nginx.com/d' /etc/apk/repositories @@ -161,7 +161,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://pkgs.nginx.com/app-protect-security-updates/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -197,7 +197,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect-x-plus/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -236,7 +236,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && gpg --dearmor -o /usr/share/keyrings/app-protect-archive-keyring.gpg /tmp/app-protect-security-updates.key \ && cp /tmp/nginx-plus.sources /etc/apt/sources.list.d/nginx-plus.sources \ && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && apt-get purge --auto-remove -y gpg \ && mkdir -p /etc/nginx/reporting/ \ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ @@ -356,7 +356,7 @@ RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_s printf "%s\n" "[nginx]" "name=nginx repo" \ "baseurl=https://nginx.org/packages/mainline/centos/9/\$basearch/" \ "gpgcheck=1" "enabled=1" "module_hotfixes=true" > /etc/yum.repos.d/nginx.repo \ - && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-image-filter nginx-module-xslt \ + && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel nginx-module-image-filter nginx-module-xslt \ && rm /etc/yum.repos.d/nginx.repo; \ fi \ && ubi-clean.sh @@ -378,7 +378,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ubi-setup.sh \ - && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && ubi-clean.sh @@ -483,7 +483,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ && subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \ @@ -530,7 +530,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ ## end of duplicated code && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ From 58294904fcd343b86ce5001265355231d0e68003 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:15:53 +0100 Subject: [PATCH 19/55] remove unused config map keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 27c51c0abe..574d20d05f 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -578,15 +578,6 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has cfgParams.MainOtelLoadModule = true } - if otelEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-enabled", cfgm); exists { - if err != nil { - nl.Error(l, err) - eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) - configOk = false - } - cfgParams.MainOtelGlobalTraceEnabled = otelGlobalTraceEnabled - } - if cfgParams.MainOtelExporterEndpoint != "" { cfgParams.MainOtelLoadModule = true } From a2989388e2e9014a15f994549feec94ef6a1550f Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 14 Apr 2025 13:02:40 +0100 Subject: [PATCH 20/55] update templates to add OSS support --- internal/configs/version1/nginx-plus.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9faf04bed9..7c7b090b4b 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -149,18 +149,18 @@ http { {{- if .MainOtelLoadModule}} otel_exporter { endpoint {{ .MainOtelExporterEndpoint}}; - {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{ if .MainOtelExporterTrustedCA}} + {{- if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} } - {{ if .MainOtelServiceName}} + {{- if .MainOtelServiceName}} otel_service_name {{ .MainOtelServiceName }}; {{- end }} - {{ if .MainOtelGlobalTraceEnabled }} + {{- if .MainOtelGlobalTraceEnabled }} otel_trace on; {{- end}} {{- end}} From 2817b2f0a27cce5618da96bdfc2f321357b80728 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 11 Apr 2025 14:45:29 +0100 Subject: [PATCH 21/55] add otel-module to NIC images Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index f493bbf89f..0ebf616d98 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -14,14 +14,15 @@ ARG PACKAGE_REPO=pkgs.nginx.com FROM ghcr.io/nginx/dependencies/nginx-ubi-ppc64le:nginx-1.27.4@sha256:fff4dde599b89cb22e5cea5d8cfba8c47bcedaa8e6fa549f5fe74a89c733aa2f AS ubi-ppc64le FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.19@sha256:2a7f8451110b588b733e4cb8727a48153057b1debac5c78ef8a539ff63712fa1 AS alpine-fips-3.19 FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.21@sha256:5221dec2e33436f2586c743c7aa3ef4626c0ec54184dc3364d101036d4f4a060 AS alpine-fips-3.21 -FROM redhat/ubi9-minimal:9.5@sha256:a50731d3397a4ee28583f1699842183d4d24fadcc565c4688487af9ee4e13a44 AS ubi-minimal -FROM golang:1.24-alpine@sha256:ef18ee7117463ac1055f5a370ed18b8750f01589f13ea0b48642f5792b234044 AS golang-builder +FROM redhat/ubi9-minimal:9.5@sha256:e1c4703364c5cb58f5462575dc90345bcd934ddc45e6c32f9c162f2b5617681c AS ubi-minimal +FROM golang:1.24-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS golang-builder ############################################# Base image for Alpine ############################################# FROM nginx:1.27.4-alpine@sha256:4ff102c5d78d254a6f0da062b3cf39eaf07f01eec0927fd21e219d0af8bc0591 AS alpine -RUN printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ +RUN apk add --no-cache libcap libstdc++ \ + && printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ && apk add --no-cache libcap libstdc++ nginx-module-otel \ && sed -i -e '/nginx.org/d' /etc/apk/repositories @@ -41,6 +42,7 @@ RUN apt-get update \ && apt-get purge --auto-remove -y gnupg2 lsb-release curl \ && rm -rf /var/lib/apt/lists/* /etc/apt/preferences.d/99nginx /etc/apt/sources.list.d/nginx.list + ############################################# NGINX files ############################################# FROM scratch AS nginx-files ARG IC_VERSION @@ -457,7 +459,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode ############################################# Base image for UBI8 with NGINX Plus and App Protect WAF ############################################# -FROM redhat/ubi8@sha256:8bd1b6306f8164de7fb0974031a0f903bd3ab3e6bcab835854d3d9a1a74ea5db AS ubi-8-plus-nap +FROM redhat/ubi8@sha256:244e9858f9d8a2792a3dceb850b4fa8fdbd67babebfde42587bfa919d5d1ecef AS ubi-8-plus-nap ARG NAP_MODULES ARG NGINX_AGENT ARG NGINX_PLUS_VERSION From b492c4065b67aa03884918428a6f27cc237212e8 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:15:53 +0100 Subject: [PATCH 22/55] remove unused config map keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/version1/nginx-plus.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 7c7b090b4b..9faf04bed9 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -149,18 +149,18 @@ http { {{- if .MainOtelLoadModule}} otel_exporter { endpoint {{ .MainOtelExporterEndpoint}}; - {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{- if .MainOtelExporterTrustedCA}} + {{ if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} } - {{- if .MainOtelServiceName}} + {{ if .MainOtelServiceName}} otel_service_name {{ .MainOtelServiceName }}; {{- end }} - {{- if .MainOtelGlobalTraceEnabled }} + {{ if .MainOtelGlobalTraceEnabled }} otel_trace on; {{- end}} {{- end}} From 5337d054e91a88874eb17176e52625fc4b87595d Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 14 Apr 2025 13:02:40 +0100 Subject: [PATCH 23/55] update templates to add OSS support --- internal/configs/version1/nginx-plus.tmpl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9faf04bed9..7c7b090b4b 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -149,18 +149,18 @@ http { {{- if .MainOtelLoadModule}} otel_exporter { endpoint {{ .MainOtelExporterEndpoint}}; - {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{ if .MainOtelExporterTrustedCA}} + {{- if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} } - {{ if .MainOtelServiceName}} + {{- if .MainOtelServiceName}} otel_service_name {{ .MainOtelServiceName }}; {{- end }} - {{ if .MainOtelGlobalTraceEnabled }} + {{- if .MainOtelGlobalTraceEnabled }} otel_trace on; {{- end}} {{- end}} From 16422211539d7d977470b9c9793d3317bca82035 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:45:45 +0100 Subject: [PATCH 24/55] accept otel configmap keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 4 ---- internal/configs/version1/nginx-plus.tmpl | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 574d20d05f..2c5f5b9109 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -578,10 +578,6 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has cfgParams.MainOtelLoadModule = true } - if cfgParams.MainOtelExporterEndpoint != "" { - cfgParams.MainOtelLoadModule = true - } - if hasAppProtect { if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists { if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" { diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 7c7b090b4b..a68be5ca2e 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -145,6 +145,22 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} + + {{- if .MainOtelEnabled}} + # otel_exporter { + # endpoint {{ .MainOtelExporterEndpoint}}; + # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + {{ if .MainOtelExporterTrustedCA}} + # trusted_certificate ; + {{- end }} + {{ if .MainOtelServiceName}} + # otel_service_name {{ .MainOtelServiceName }}; + # } + {{- end }} + {{ if .MainOtelGlobalTraceEnabled }} + # otel_trace on; + {{- end}} + {{- end}} {{- if .MainOtelLoadModule}} otel_exporter { From 8bcbf71476b48929c7da08626837ac21d1af8168 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 11 Apr 2025 14:45:29 +0100 Subject: [PATCH 25/55] add otel-module to NIC images Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 0ebf616d98..958b54bfec 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -21,8 +21,7 @@ FROM golang:1.24-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754 ############################################# Base image for Alpine ############################################# FROM nginx:1.27.4-alpine@sha256:4ff102c5d78d254a6f0da062b3cf39eaf07f01eec0927fd21e219d0af8bc0591 AS alpine -RUN apk add --no-cache libcap libstdc++ \ - && printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ +RUN printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ && apk add --no-cache libcap libstdc++ nginx-module-otel \ && sed -i -e '/nginx.org/d' /etc/apk/repositories From d6bd551435253c544cc6fb1d3aa2e0be9eca9fdb Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 14 Apr 2025 12:15:53 +0100 Subject: [PATCH 26/55] remove unused config map keys Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/version1/nginx-plus.tmpl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index a68be5ca2e..490c97b0fc 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -145,20 +145,23 @@ http { {{- if .SSLDHParam}} ssl_dhparam {{.SSLDHParam}}; {{- end}} - - {{- if .MainOtelEnabled}} - # otel_exporter { - # endpoint {{ .MainOtelExporterEndpoint}}; - # header {{ .MainOtelExporterHeaderName }} {{ .MainOtelExporterHeaderValue }}; + + {{- if .MainOtelLoadModule}} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint}}; + {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} {{ if .MainOtelExporterTrustedCA}} # trusted_certificate ; {{- end }} + } + {{ if .MainOtelServiceName}} - # otel_service_name {{ .MainOtelServiceName }}; - # } + otel_service_name {{ .MainOtelServiceName }}; {{- end }} {{ if .MainOtelGlobalTraceEnabled }} - # otel_trace on; + otel_trace on; {{- end}} {{- end}} From c7fee53c5a46a5b429434a2e069ab66e5babf4f7 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 29 Apr 2025 15:27:39 +0100 Subject: [PATCH 27/55] add CA to Otel exporter on startup --- internal/configs/version1/nginx-plus.tmpl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 490c97b0fc..9fa8cc6ba0 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -146,14 +146,14 @@ http { ssl_dhparam {{.SSLDHParam}}; {{- end}} - {{- if .MainOtelLoadModule}} + {{- if .MainOtelLoadModule }} otel_exporter { - endpoint {{ .MainOtelExporterEndpoint}}; - {{ if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + endpoint {{ .MainOtelExporterEndpoint }}; + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{ if .MainOtelExporterTrustedCA}} - # trusted_certificate ; + {{- if .MainOtelExporterTrustedCA }} + trusted_certificate /etc/nginx/secrets/{{ .MainOtelExporterTrustedCA }}; {{- end }} } From f562f9e331825e5d5292ba31a4e0013ca651a119 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Thu, 15 May 2025 15:57:00 +0100 Subject: [PATCH 28/55] Remove cert processing, rename configmap keys, fix tests Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- cmd/nginx-ingress/main.go | 26 +--- internal/configs/config_params.go | 2 +- internal/configs/configmaps.go | 130 +++++++++------- internal/configs/configmaps_test.go | 176 ++++++++++++++++++++++ internal/configs/configurator.go | 5 +- internal/configs/version1/nginx-plus.tmpl | 22 --- internal/configs/version1/nginx.tmpl | 3 - internal/k8s/controller.go | 61 ++------ 8 files changed, 270 insertions(+), 155 deletions(-) diff --git a/cmd/nginx-ingress/main.go b/cmd/nginx-ingress/main.go index 262647ffc5..30f4cfa0cf 100644 --- a/cmd/nginx-ingress/main.go +++ b/cmd/nginx-ingress/main.go @@ -166,7 +166,7 @@ func main() { logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) } - if err := processMgmtTrustedCertSecret(kubeClient, nginxManager, mgmtCfgParams, controllerNamespace); err != nil { + if err := processTrustedCertSecret(kubeClient, nginxManager, mgmtCfgParams, controllerNamespace); err != nil { logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) } @@ -189,7 +189,6 @@ func main() { if err != nil { logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) } - globalConfigurationValidator := createGlobalConfigurationValidator() mustProcessGlobalConfiguration(ctx) @@ -197,10 +196,6 @@ func main() { cfgParams := configs.NewDefaultConfigParams(ctx, *nginxPlus) cfgParams = processConfigMaps(kubeClient, cfgParams, nginxManager, templateExecutor, eventRecorder) - if err := processOtelTrustedCertSecret(kubeClient, nginxManager, cfgParams, controllerNamespace); err != nil { - logEventAndExit(ctx, eventRecorder, pod, secretErrorReason, err) - } - staticCfgParams := &configs.StaticConfigParams{ DisableIPV6: *disableIPV6, DefaultHTTPListenerPort: *defaultHTTPListenerPort, @@ -376,7 +371,7 @@ func processClientAuthSecret(kubeClient *kubernetes.Clientset, nginxManager ngin return nil } -func processMgmtTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, mgmtCfgParams *configs.MGMTConfigParams, controllerNamespace string) error { +func processTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, mgmtCfgParams *configs.MGMTConfigParams, controllerNamespace string) error { if mgmtCfgParams.Secrets.TrustedCert == "" { return nil } @@ -397,23 +392,6 @@ func processMgmtTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager return nil } -func processOtelTrustedCertSecret(kubeClient *kubernetes.Clientset, nginxManager nginx.Manager, cfgParams *configs.ConfigParams, controllerNamespace string) error { - if cfgParams.MainOtelExporterTrustedCA == "" { - return nil - } - - trustedCertSecretNsName := controllerNamespace + "/" + cfgParams.MainOtelExporterTrustedCA - - secret, err := getAndValidateSecret(kubeClient, trustedCertSecretNsName, secrets.SecretTypeCA) - if err != nil { - return fmt.Errorf("error trying to get the trusted cert secret %v: %w", trustedCertSecretNsName, err) - } - - caBytes, _ := configs.GenerateCAFileContent(secret) - nginxManager.CreateSecret(fmt.Sprintf("%s-%s-%s", controllerNamespace, cfgParams.MainOtelExporterTrustedCA, configs.CACrtKey), caBytes, nginx.ReadWriteOnlyFileMode) - return nil -} - func mustCreateConfigAndKubeClient(ctx context.Context) (*rest.Config, *kubernetes.Clientset) { l := nl.LoggerFromContext(ctx) var config *rest.Config diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index 5c904ef5e3..44f65d3e3a 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -35,7 +35,7 @@ type ConfigParams struct { MainLogFormatEscaping string MainMainSnippets []string MainOtelLoadModule bool - MainOtelGlobalTraceEnabled bool + MainOtelTraceInHTTP bool MainOtelExporterEndpoint string MainOtelExporterTrustedCA string MainOtelExporterHeaderName string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 2c5f5b9109..148f8e9a62 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -530,52 +530,9 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has } } - if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { - otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) - if otelExporterEndpoint != "" { - cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint - } - } - - if otelExporterTrustedCA, exists := cfgm.Data["otel-exporter-trusted-ca"]; exists { - otelExporterTrustedCA = strings.TrimSpace(otelExporterTrustedCA) - if otelExporterTrustedCA != "" { - cfgParams.MainOtelExporterTrustedCA = otelExporterTrustedCA - } - } - - if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { - otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) - if otelExporterHeaderName != "" { - cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName - } - } - - if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists { - otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue) - if otelExporterHeaderValue != "" { - cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue - } - } - - if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists { - otelServiceName = strings.TrimSpace(otelServiceName) - if otelServiceName != "" { - cfgParams.MainOtelServiceName = otelServiceName - } - } - - if otelGlobalTraceEnabled, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-global-trace-enabled", cfgm); exists { - if err != nil { - nl.Error(l, err) - eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) - configOk = false - } - cfgParams.MainOtelGlobalTraceEnabled = otelGlobalTraceEnabled - } - - if cfgParams.MainOtelExporterEndpoint != "" { - cfgParams.MainOtelLoadModule = true + _, otelErr := parseConfigMapOpenTelemetry(l, cfgm, cfgParams, eventLog) + if otelErr != nil { + configOk = false } if hasAppProtect { @@ -788,6 +745,79 @@ func parseConfigMapZoneSync(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams *Confi return &cfgParams.ZoneSync, nil } +//nolint:gocyclo +func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams *ConfigParams, eventLog record.EventRecorder) (*ConfigParams, error) { + if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { + otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) + if otelExporterEndpoint != "" { + cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint + } + } + + if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { + otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) + if otelExporterHeaderName != "" { + cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName + } + } + + if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists { + otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue) + if otelExporterHeaderValue != "" { + cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue + } + } + + if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists { + otelServiceName = strings.TrimSpace(otelServiceName) + if otelServiceName != "" { + cfgParams.MainOtelServiceName = otelServiceName + } + } + + otelValid := true + + if otelTraceInHTTP, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-trace-in-http", cfgm); exists { + if err != nil { + nl.Error(l, err) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) + otelValid = false + } + cfgParams.MainOtelTraceInHTTP = otelTraceInHTTP + } + + if (cfgParams.MainOtelExporterHeaderName != "" && cfgParams.MainOtelExporterHeaderValue == "") || + (cfgParams.MainOtelExporterHeaderName == "" && cfgParams.MainOtelExporterHeaderValue != "") { + errorText := "Both 'otel-exporter-header-name' and 'otel-exporter-header-value' must be set or neither" + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + otelValid = false + } + + if cfgParams.MainOtelExporterEndpoint != "" { + cfgParams.MainOtelLoadModule = true + } + + if cfgParams.MainOtelExporterEndpoint == "" && + (cfgParams.MainOtelExporterTrustedCA != "" || + cfgParams.MainOtelExporterHeaderName != "" || + cfgParams.MainOtelExporterHeaderValue != "" || + cfgParams.MainOtelServiceName != "" || + cfgParams.MainOtelTraceInHTTP) { + errorText := "ConfigMap key 'otel-exporter-endpoint' is required when other otel fields are set" + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + otelValid = false + cfgParams.MainOtelTraceInHTTP = false + } + + if !otelValid { + return nil, errors.New("invalid OpenTelemetry configuration") + } + + return cfgParams, nil +} + // ParseMGMTConfigMap parses the mgmt block ConfigMap into MGMTConfigParams. // //nolint:gocyclo @@ -940,11 +970,6 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config ResolverValid: config.ZoneSync.ResolverValid, } - mainOtelExporterTrustedCA := "" - if config.MainOtelExporterTrustedCA != "" { - mainOtelExporterTrustedCA = fmt.Sprintf("%s-%s-%s", os.Getenv("POD_NAMESPACE"), config.MainOtelExporterTrustedCA, CACrtKey) - } - nginxCfg := &version1.MainConfig{ AccessLog: config.MainAccessLog, DefaultServerAccessLogOff: config.DefaultServerAccessLogOff, @@ -967,9 +992,8 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config NginxStatusAllowCIDRs: staticCfgParams.NginxStatusAllowCIDRs, NginxStatusPort: staticCfgParams.NginxStatusPort, MainOtelLoadModule: config.MainOtelLoadModule, - MainOtelGlobalTraceEnabled: config.MainOtelGlobalTraceEnabled, + MainOtelGlobalTraceEnabled: config.MainOtelTraceInHTTP, MainOtelExporterEndpoint: config.MainOtelExporterEndpoint, - MainOtelExporterTrustedCA: mainOtelExporterTrustedCA, MainOtelExporterHeaderName: config.MainOtelExporterHeaderName, MainOtelExporterHeaderValue: config.MainOtelExporterHeaderValue, MainOtelServiceName: config.MainOtelServiceName, diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index 517c384780..e62e63c431 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -1376,6 +1376,182 @@ func TestParseZoneSyncResolverIPV6MapResolverIPV6(t *testing.T) { }) } } +func TestOpenTelemetryConfigurationSuccess(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + expectedLoadModule bool + expectedExporterEndpoint string + //expectedExporterTrustedCA string + expectedExporterHeaderName string + expectedExporterHeaderValue string + expectedServiceName string + expectedTraceInHTTP bool + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-service-name": "nginx-ingress-controller", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller", + msg: "endpoint set, minimal config", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-trusted-ca": "otel-ca-secret", + "otel-exporter-header-name": "X-Custom-Header", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller", + "otel-trace-in-http": "true", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "X-Custom-Header", + expectedExporterHeaderValue: "custom-value", + expectedServiceName: "nginx-ingress-controller", + expectedTraceInHTTP: true, + msg: "endpoint set, full config", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{}, + }, + expectedLoadModule: false, + expectedExporterEndpoint: "", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "", + expectedTraceInHTTP: false, + msg: "no config", + }, + } + + isPlus := true + hasAppProtect := false + hasAppProtectDos := false + hasTLSPassthrough := false + const expectedConfigOk = true + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + result, configOk := ParseConfigMap(context.Background(), test.configMap, isPlus, + hasAppProtect, hasAppProtectDos, hasTLSPassthrough, makeEventLogger()) + if configOk != expectedConfigOk { + t.Errorf("configOk: want %v, got %v", expectedConfigOk, configOk) + } + if result.MainOtelLoadModule != test.expectedLoadModule { + t.Errorf("MainOtelLoadModule: want %v, got %v", test.expectedLoadModule, result.MainOtelLoadModule) + } + if result.MainOtelExporterEndpoint != test.expectedExporterEndpoint { + t.Errorf("MainOtelExporterEndpoint: want %q, got %q", test.expectedExporterEndpoint, result.MainOtelExporterEndpoint) + } + if result.MainOtelExporterHeaderName != test.expectedExporterHeaderName { + t.Errorf("MainOtelExporterHeaderName: want %q, got %q", test.expectedExporterHeaderName, result.MainOtelExporterHeaderName) + } + if result.MainOtelExporterHeaderValue != test.expectedExporterHeaderValue { + t.Errorf("MainOtelExporterHeaderValue: want %q, got %q", test.expectedExporterHeaderValue, result.MainOtelExporterHeaderValue) + } + if result.MainOtelServiceName != test.expectedServiceName { + t.Errorf("MainOtelServiceName: want %q, got %q", test.expectedServiceName, result.MainOtelServiceName) + } + if result.MainOtelTraceInHTTP != test.expectedTraceInHTTP { + t.Errorf("MainOtelTraceInHTTP: want %v, got %v", test.expectedTraceInHTTP, result.MainOtelTraceInHTTP) + } + }) + } +} + +func TestOpenTelemetryConfigurationInvalid(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + expectedLoadModule bool + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": " ", + "otel-service-name": "nginx-ingress-controller", + }, + }, + expectedLoadModule: false, + msg: "empty endpoint, service name set", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-name": "X-Custom-Header", + "otel-service-name": "nginx-ingress-controller", + }, + }, + expectedLoadModule: true, + msg: "endpoint set, header name only", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller", + }, + }, + expectedLoadModule: true, + msg: "endpoint set, header value only", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-header-name": "X-Custom-Header", + "otel-exporter-header-value": "custom-value", + }, + }, + expectedLoadModule: false, + msg: "no endpoint, headers set", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-service-name": "nginx-ingress-controller", + "otel-trace-in-http": "invalid", + }, + }, + expectedLoadModule: true, + msg: "endpoint set, invalid trace flag", + }, + } + + isPlus := true + hasAppProtect := false + hasAppProtectDos := false + hasTLSPassthrough := false + expectedConfigOk := false + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + result, configOk := ParseConfigMap(context.Background(), test.configMap, isPlus, + hasAppProtect, hasAppProtectDos, hasTLSPassthrough, makeEventLogger()) + if configOk != expectedConfigOk { + t.Errorf("configOk: want %v, got %v", expectedConfigOk, configOk) + } + if result.MainOtelLoadModule != test.expectedLoadModule { + t.Errorf("MainOtelLoadModule: want %v, got %v", test.expectedLoadModule, result.MainOtelLoadModule) + } + }) + } +} func makeEventLogger() record.EventRecorder { return record.NewFakeRecorder(1024) diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index ba370848e8..110ab16d6f 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -826,11 +826,8 @@ func generateTLSPassthroughHostsConfig(tlsPassthroughPairs map[string]tlsPassthr // AddOrUpdateCASecret writes the secret content to disk returning the files added/updated func (cnf *Configurator) AddOrUpdateCASecret(secret *api_v1.Secret, crtFileName, crlFileName string) string { crtData, crlData := GenerateCAFileContent(secret) - crlFilePath := "" crtFilePath := cnf.nginxManager.CreateSecret(crtFileName, crtData, nginx.ReadWriteOnlyFileMode) - if len(crlData) > 0 { - crlFilePath = cnf.nginxManager.CreateSecret(crlFileName, crlData, nginx.ReadWriteOnlyFileMode) - } + crlFilePath := cnf.nginxManager.CreateSecret(crlFileName, crlData, nginx.ReadWriteOnlyFileMode) return fmt.Sprintf("%s %s", crtFilePath, crlFilePath) } diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9fa8cc6ba0..9355d503e6 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -152,9 +152,6 @@ http { {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{- if .MainOtelExporterTrustedCA }} - trusted_certificate /etc/nginx/secrets/{{ .MainOtelExporterTrustedCA }}; - {{- end }} } {{ if .MainOtelServiceName}} @@ -165,25 +162,6 @@ http { {{- end}} {{- end}} - {{- if .MainOtelLoadModule}} - otel_exporter { - endpoint {{ .MainOtelExporterEndpoint}}; - {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} - header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; - {{- end }} - {{- if .MainOtelExporterTrustedCA}} - # trusted_certificate ; - {{- end }} - } - - {{- if .MainOtelServiceName}} - otel_service_name {{ .MainOtelServiceName }}; - {{- end }} - {{- if .MainOtelGlobalTraceEnabled }} - otel_trace on; - {{- end}} - {{- end}} - {{- if .MainOtelLoadModule }} otel_exporter { endpoint {{ .MainOtelExporterEndpoint }}; diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index 2d1822c8ee..a5f8d7760b 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -114,9 +114,6 @@ http { {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; {{- end }} - {{- if .MainOtelExporterTrustedCA }} - trusted_certificate /etc/nginx/secrets/{{ .MainOtelExporterTrustedCA }}; - {{- end }} } {{- if .MainOtelServiceName}} diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 151358649b..3435e07de1 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -102,12 +102,11 @@ type podEndpoint struct { } type specialSecrets struct { - defaultServerSecret string - wildcardTLSSecret string - licenseSecret string - clientAuthSecret string - mgmtTrustedCertSecret string - otelTrustedCertSecret string + defaultServerSecret string + wildcardTLSSecret string + licenseSecret string + clientAuthSecret string + trustedCertSecret string } type controllerMetadata struct { @@ -248,19 +247,14 @@ type NewLoadBalancerControllerInput struct { // NewLoadBalancerController creates a controller func NewLoadBalancerController(input NewLoadBalancerControllerInput) *LoadBalancerController { - otelTrustedCertSecret := "" - if input.NginxConfigurator != nil && input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA != "" { - otelTrustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.CfgParams.MainOtelExporterTrustedCA) - } specialSecrets := specialSecrets{ - defaultServerSecret: input.DefaultServerSecret, - wildcardTLSSecret: input.WildcardTLSSecret, - otelTrustedCertSecret: otelTrustedCertSecret, + defaultServerSecret: input.DefaultServerSecret, + wildcardTLSSecret: input.WildcardTLSSecret, } if input.IsNginxPlus { specialSecrets.licenseSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.License) specialSecrets.clientAuthSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.ClientAuth) - specialSecrets.mgmtTrustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.TrustedCert) + specialSecrets.trustedCertSecret = fmt.Sprintf("%s/%s", input.ControllerNamespace, input.NginxConfigurator.MgmtCfgParams.Secrets.TrustedCert) } lbc := &LoadBalancerController{ client: input.KubeClient, @@ -930,7 +924,7 @@ func (lbc *LoadBalancerController) updateAllConfigs() { if _, hasCRL := secret.Data[configs.CACrlKey]; hasCRL { lbc.configurator.MgmtCfgParams.Secrets.TrustedCRL = secret.Name } - lbc.specialSecrets.mgmtTrustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name) + lbc.specialSecrets.trustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name) lbc.handleSpecialSecretUpdate(secret, reloadNginx) } // update special ClientAuth secret in mgmtConfigParams @@ -943,15 +937,6 @@ func (lbc *LoadBalancerController) updateAllConfigs() { lbc.handleSpecialSecretUpdate(secret, reloadNginx) } } - // update special Otel CA secret in configParams - if cfgParams.MainOtelExporterTrustedCA != "" { - secret, err := lbc.client.CoreV1().Secrets(lbc.configMap.GetNamespace()).Get(context.TODO(), cfgParams.MainOtelExporterTrustedCA, meta_v1.GetOptions{}) - if err != nil { - nl.Errorf(lbc.Logger, "secret %s/%s: %v", lbc.configMap.GetNamespace(), cfgParams.MainOtelExporterTrustedCA, err) - } - lbc.specialSecrets.otelTrustedCertSecret = fmt.Sprintf("%s/%s", secret.Namespace, secret.Name) - lbc.handleSpecialSecretUpdate(secret, reloadNginx) - } resources := lbc.configuration.GetResources() nl.Debugf(lbc.Logger, "Updating %v resources", len(resources)) resourceExes := lbc.createExtendedResources(resources) @@ -1858,9 +1843,7 @@ func (lbc *LoadBalancerController) isSpecialSecret(secretName string) bool { return true case lbc.specialSecrets.clientAuthSecret: return true - case lbc.specialSecrets.mgmtTrustedCertSecret: - return true - case lbc.specialSecrets.otelTrustedCertSecret: + case lbc.specialSecrets.trustedCertSecret: return true default: return false @@ -1935,12 +1918,7 @@ func (lbc *LoadBalancerController) handleSpecialSecretUpdate(secret *api_v1.Secr if ok := lbc.performNGINXReload(secret); !ok { return } - case lbc.specialSecrets.mgmtTrustedCertSecret: - lbc.updateAllConfigs() - if ok := lbc.performNGINXReload(secret); !ok { - return - } - case lbc.specialSecrets.otelTrustedCertSecret: + case lbc.specialSecrets.trustedCertSecret: lbc.updateAllConfigs() if ok := lbc.performNGINXReload(secret); !ok { return @@ -1966,12 +1944,7 @@ func (lbc *LoadBalancerController) writeSpecialSecrets(secret *api_v1.Secret, sp return false } case secrets.SecretTypeCA: - if lbc.specialSecrets.mgmtTrustedCertSecret != "" { - lbc.configurator.AddOrUpdateCASecret(secret, fmt.Sprintf("mgmt/%s", configs.CACrtKey), fmt.Sprintf("mgmt/%s", configs.CACrlKey)) - } - if lbc.specialSecrets.otelTrustedCertSecret != "" { - lbc.configurator.AddOrUpdateCASecret(secret, fmt.Sprintf("%s-%s-%s", lbc.metadata.namespace, lbc.configurator.CfgParams.MainOtelExporterTrustedCA, configs.CACrtKey), "") - } + lbc.configurator.AddOrUpdateCASecret(secret, fmt.Sprintf("mgmt/%s", configs.CACrtKey), fmt.Sprintf("mgmt/%s", configs.CACrlKey)) case api_v1.SecretTypeTLS: // if the secret name matches the specified if secretNsName == mgmtClientAuthNamespaceName { @@ -2008,7 +1981,7 @@ func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, return false } } - if secretNsName == lbc.specialSecrets.mgmtTrustedCertSecret { + if secretNsName == lbc.specialSecrets.trustedCertSecret { err := secrets.ValidateCASecret(secret) if err != nil { nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err) @@ -2024,14 +1997,6 @@ func (lbc *LoadBalancerController) specialSecretValidation(secretNsName string, return false } } - if secretNsName == lbc.specialSecrets.otelTrustedCertSecret { - err := secrets.ValidateCASecret(secret) - if err != nil { - nl.Errorf(lbc.Logger, "Couldn't validate the special Secret %v: %v", secretNsName, err) - lbc.recorder.Eventf(lbc.metadata.pod, api_v1.EventTypeWarning, nl.EventReasonRejected, "the special Secret %v was rejected, using the previous version: %v", secretNsName, err) - return false - } - } return true } From c23684e002f2836ed0ee68b9d1fafb123268e8c4 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Thu, 15 May 2025 16:20:12 +0100 Subject: [PATCH 29/55] fix rebase Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- build/Dockerfile | 7 +++---- internal/configs/config_params.go | 1 - internal/configs/configmaps.go | 3 +-- internal/configs/configmaps_test.go | 7 +++---- internal/configs/version1/config.go | 1 - 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 958b54bfec..2e54241801 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -14,8 +14,8 @@ ARG PACKAGE_REPO=pkgs.nginx.com FROM ghcr.io/nginx/dependencies/nginx-ubi-ppc64le:nginx-1.27.4@sha256:fff4dde599b89cb22e5cea5d8cfba8c47bcedaa8e6fa549f5fe74a89c733aa2f AS ubi-ppc64le FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.19@sha256:2a7f8451110b588b733e4cb8727a48153057b1debac5c78ef8a539ff63712fa1 AS alpine-fips-3.19 FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.21@sha256:5221dec2e33436f2586c743c7aa3ef4626c0ec54184dc3364d101036d4f4a060 AS alpine-fips-3.21 -FROM redhat/ubi9-minimal:9.5@sha256:e1c4703364c5cb58f5462575dc90345bcd934ddc45e6c32f9c162f2b5617681c AS ubi-minimal -FROM golang:1.24-alpine@sha256:7772cb5322baa875edd74705556d08f0eeca7b9c4b5367754ce3f2f00041ccee AS golang-builder +FROM redhat/ubi9-minimal:9.5@sha256:a50731d3397a4ee28583f1699842183d4d24fadcc565c4688487af9ee4e13a44 AS ubi-minimal +FROM golang:1.24-alpine@sha256:ef18ee7117463ac1055f5a370ed18b8750f01589f13ea0b48642f5792b234044 AS golang-builder ############################################# Base image for Alpine ############################################# @@ -27,7 +27,7 @@ RUN printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^ ############################################# Base image for Debian ############################################# -FROM nginx:1.27.4@sha256:124b44bfc9ccd1f3cedf4b592d4d1e8bddb78b51ec2ed5056c52d3692baebc19 AS debian +FROM nginx:1.27.4@sha256:09369da6b10306312cd908661320086bf87fbae1b6b0c49a1f50ba531fef2eab AS debian RUN apt-get update \ && apt-get install --no-install-recommends --no-install-suggests -y \ @@ -41,7 +41,6 @@ RUN apt-get update \ && apt-get purge --auto-remove -y gnupg2 lsb-release curl \ && rm -rf /var/lib/apt/lists/* /etc/apt/preferences.d/99nginx /etc/apt/sources.list.d/nginx.list - ############################################# NGINX files ############################################# FROM scratch AS nginx-files ARG IC_VERSION diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index 44f65d3e3a..c73310056d 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -37,7 +37,6 @@ type ConfigParams struct { MainOtelLoadModule bool MainOtelTraceInHTTP bool MainOtelExporterEndpoint string - MainOtelExporterTrustedCA string MainOtelExporterHeaderName string MainOtelExporterHeaderValue string MainOtelServiceName string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 148f8e9a62..54c7cfc39c 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -799,8 +799,7 @@ func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams * } if cfgParams.MainOtelExporterEndpoint == "" && - (cfgParams.MainOtelExporterTrustedCA != "" || - cfgParams.MainOtelExporterHeaderName != "" || + (cfgParams.MainOtelExporterHeaderName != "" || cfgParams.MainOtelExporterHeaderValue != "" || cfgParams.MainOtelServiceName != "" || cfgParams.MainOtelTraceInHTTP) { diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index e62e63c431..ddc57d0b13 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -1379,10 +1379,9 @@ func TestParseZoneSyncResolverIPV6MapResolverIPV6(t *testing.T) { func TestOpenTelemetryConfigurationSuccess(t *testing.T) { t.Parallel() tests := []struct { - configMap *v1.ConfigMap - expectedLoadModule bool - expectedExporterEndpoint string - //expectedExporterTrustedCA string + configMap *v1.ConfigMap + expectedLoadModule bool + expectedExporterEndpoint string expectedExporterHeaderName string expectedExporterHeaderValue string expectedServiceName string diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index a42730058a..106470b865 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -243,7 +243,6 @@ type MainConfig struct { MainOtelLoadModule bool MainOtelGlobalTraceEnabled bool MainOtelExporterEndpoint string - MainOtelExporterTrustedCA string MainOtelExporterHeaderName string MainOtelExporterHeaderValue string MainOtelServiceName string From 166420cfc41b8face9988c3a2ada59e3f9709413 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Thu, 15 May 2025 16:23:43 +0100 Subject: [PATCH 30/55] fix linting Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index ddc57d0b13..0d5c46ad80 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -1376,6 +1376,7 @@ func TestParseZoneSyncResolverIPV6MapResolverIPV6(t *testing.T) { }) } } + func TestOpenTelemetryConfigurationSuccess(t *testing.T) { t.Parallel() tests := []struct { From aeaf2817d49b9e8c033ef35afd1f7494c07c98b8 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Fri, 16 May 2025 09:51:10 +0100 Subject: [PATCH 31/55] add template test Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps_test.go | 16 +- .../version1/__snapshots__/template_test.snap | 148 ++++++++++++++++++ internal/configs/version1/template_test.go | 39 +++++ 3 files changed, 195 insertions(+), 8 deletions(-) diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index 0d5c46ad80..1559ede49d 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -1393,14 +1393,14 @@ func TestOpenTelemetryConfigurationSuccess(t *testing.T) { configMap: &v1.ConfigMap{ Data: map[string]string{ "otel-exporter-endpoint": "https://otel-collector:4317", - "otel-service-name": "nginx-ingress-controller", + "otel-service-name": "nginx-ingress-controller:nginx", }, }, expectedLoadModule: true, expectedExporterEndpoint: "https://otel-collector:4317", expectedExporterHeaderName: "", expectedExporterHeaderValue: "", - expectedServiceName: "nginx-ingress-controller", + expectedServiceName: "nginx-ingress-controller:nginx", msg: "endpoint set, minimal config", }, { @@ -1410,7 +1410,7 @@ func TestOpenTelemetryConfigurationSuccess(t *testing.T) { "otel-exporter-trusted-ca": "otel-ca-secret", "otel-exporter-header-name": "X-Custom-Header", "otel-exporter-header-value": "custom-value", - "otel-service-name": "nginx-ingress-controller", + "otel-service-name": "nginx-ingress-controller:nginx", "otel-trace-in-http": "true", }, }, @@ -1418,7 +1418,7 @@ func TestOpenTelemetryConfigurationSuccess(t *testing.T) { expectedExporterEndpoint: "https://otel-collector:4317", expectedExporterHeaderName: "X-Custom-Header", expectedExporterHeaderValue: "custom-value", - expectedServiceName: "nginx-ingress-controller", + expectedServiceName: "nginx-ingress-controller:nginx", expectedTraceInHTTP: true, msg: "endpoint set, full config", }, @@ -1482,7 +1482,7 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { configMap: &v1.ConfigMap{ Data: map[string]string{ "otel-exporter-endpoint": " ", - "otel-service-name": "nginx-ingress-controller", + "otel-service-name": "nginx-ingress-controller:nginx", }, }, expectedLoadModule: false, @@ -1493,7 +1493,7 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { Data: map[string]string{ "otel-exporter-endpoint": "https://otel-collector:4317", "otel-exporter-header-name": "X-Custom-Header", - "otel-service-name": "nginx-ingress-controller", + "otel-service-name": "nginx-ingress-controller:nginx", }, }, expectedLoadModule: true, @@ -1504,7 +1504,7 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { Data: map[string]string{ "otel-exporter-endpoint": "https://otel-collector:4317", "otel-exporter-header-value": "custom-value", - "otel-service-name": "nginx-ingress-controller", + "otel-service-name": "nginx-ingress-controller:nginx", }, }, expectedLoadModule: true, @@ -1524,7 +1524,7 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { configMap: &v1.ConfigMap{ Data: map[string]string{ "otel-exporter-endpoint": "https://otel-collector:4317", - "otel-service-name": "nginx-ingress-controller", + "otel-service-name": "nginx-ingress-controller:nginx", "otel-trace-in-http": "invalid", }, }, diff --git a/internal/configs/version1/__snapshots__/template_test.snap b/internal/configs/version1/__snapshots__/template_test.snap index 4a2a657e12..66a7b6f4b7 100644 --- a/internal/configs/version1/__snapshots__/template_test.snap +++ b/internal/configs/version1/__snapshots__/template_test.snap @@ -4263,6 +4263,154 @@ stream { --- +[TestExecuteTemplate_ForMainForNGINXWithOtel - 1] +worker_processes ; + +daemon off; + +error_log stderr ; +pid /var/lib/nginx/nginx.pid; +load_module modules/ngx_otel_module.so; +load_module modules/ngx_fips_check_module.so; + +load_module modules/ngx_http_js_module.so; + +events { + worker_connections ; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + map_hash_max_size ; + map_hash_bucket_size ; + + js_import /etc/nginx/njs/apikey_auth.js; + js_set $apikey_auth_hash apikey_auth.hash; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $upstream_trailer_grpc_status $grpc_status { + default $upstream_trailer_grpc_status; + '' $sent_http_grpc_status; + } + + access_log ; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout ; + keepalive_requests 0; + + #gzip on; + + server_names_hash_max_size ; + + + variables_hash_bucket_size 0; + variables_hash_max_size 0; + + map $request_uri $request_uri_no_args { + "~^(?P[^?]*)(\?.*)?$" $path; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + map $http_upgrade $vs_connection_header { + default upgrade; + '' $default_connection_header; + } + otel_exporter { + endpoint https://otel-collector:4317; + header X-Custom-Header "custom-value"; + } + + + otel_service_name nginx-ingress-controller:nginx; + + otel_trace on; + + + + + server { + # required to support the Websocket protocol in VirtualServer/VirtualServerRoutes + set $default_connection_header ""; + set $resource_type ""; + set $resource_name ""; + set $resource_namespace ""; + set $service ""; + + listen 0 default_server;listen [::]:0 default_server; + listen 0 ssl default_server; + listen [::]:0 ssl default_server; + ssl_certificate /etc/nginx/secrets/default; + ssl_certificate_key /etc/nginx/secrets/default; + + server_name _; + server_tokens ""; + + location / { + return ; + } + } + + # NGINX Plus API over unix socket + server { + listen unix:/var/lib/nginx/nginx-plus-api.sock; + access_log off; + + # $config_version_mismatch is defined in /etc/nginx/config-version.conf + location /configVersionCheck { + if ($config_version_mismatch) { + return 503; + } + return 200; + } + + location /api { + api write=on; + } + } + + include /etc/nginx/config-version.conf; + include /etc/nginx/conf.d/*.conf; + + server { + listen unix:/var/lib/nginx/nginx-418-server.sock; + access_log off; + + return 418; + } +} + +stream { + log_format stream-main '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$ssl_preread_server_name"'; + + access_log /dev/stdout stream-main; + + + + map_hash_max_size ; + + include /etc/nginx/stream-conf.d/*.conf; +} + +mgmt { + license_token /license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; +} + +--- + [TestExecuteTemplate_ForMainForNGINXWithZoneSyncEnabledCustomPort - 1] worker_processes ; diff --git a/internal/configs/version1/template_test.go b/internal/configs/version1/template_test.go index e21da7ec80..05430f1945 100644 --- a/internal/configs/version1/template_test.go +++ b/internal/configs/version1/template_test.go @@ -1080,6 +1080,36 @@ func TestExecuteTemplate_ForMainForNGINXWithZoneSyncEnabledCustomResolverAddress snaps.MatchSnapshot(t, buf.String()) } +func TestExecuteTemplate_ForMainForNGINXWithOtel(t *testing.T) { + t.Parallel() + + tmpl := newNGINXPlusMainTmpl(t) + buf := &bytes.Buffer{} + + err := tmpl.Execute(buf, mainCfgWithOTel) + t.Log(buf.String()) + + if err != nil { + t.Fatalf("Failed to write template %v", err) + } + + wantDirectives := []string{ + "otel_exporter {", + "endpoint https://otel-collector:4317;", + "header X-Custom-Header \"custom-value\";", + "otel_service_name nginx-ingress-controller:nginx;", + "otel_trace on;", + } + + mainConf := buf.String() + for _, want := range wantDirectives { + if !strings.Contains(mainConf, want) { + t.Errorf("want %q in generated config", want) + } + } + snaps.MatchSnapshot(t, buf.String()) +} + func TestExecuteTemplate_ForIngressForNGINXWithProxySetHeadersAnnotationWithDefaultValue(t *testing.T) { t.Parallel() @@ -2589,6 +2619,15 @@ var ( }, } + mainCfgWithOTel = MainConfig{ + MainOtelLoadModule: true, + MainOtelGlobalTraceEnabled: true, + MainOtelExporterEndpoint: "https://otel-collector:4317", + MainOtelExporterHeaderName: "X-Custom-Header", + MainOtelExporterHeaderValue: "custom-value", + MainOtelServiceName: "nginx-ingress-controller:nginx", + } + // Vars for Mergable Ingress Master - Minion tests coffeeUpstreamNginxPlus = Upstream{ From 0052ce1d86604d2625fc52c587a4eedd16615561 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 11:40:46 +0100 Subject: [PATCH 32/55] add otel pytests --- tests/Makefile | 2 +- tests/data/otel/configmap-with-all.yaml | 11 + tests/data/otel/configmap-with-endpoint.yaml | 7 + tests/data/otel/configmap-with-header.yaml | 9 + .../otel/configmap-with-only-header-name.yaml | 8 + .../configmap-with-only-header-value.yaml | 8 + .../data/otel/configmap-with-otel-trace.yaml | 8 + .../otel/configmap-with-service-name.yaml | 8 + tests/data/otel/default-configmap.yaml | 6 + tests/suite/test_otel.py | 432 ++++++++++++++++++ tests/suite/utils/resources_utils.py | 4 +- 11 files changed, 500 insertions(+), 3 deletions(-) create mode 100644 tests/data/otel/configmap-with-all.yaml create mode 100644 tests/data/otel/configmap-with-endpoint.yaml create mode 100644 tests/data/otel/configmap-with-header.yaml create mode 100644 tests/data/otel/configmap-with-only-header-name.yaml create mode 100644 tests/data/otel/configmap-with-only-header-value.yaml create mode 100644 tests/data/otel/configmap-with-otel-trace.yaml create mode 100644 tests/data/otel/configmap-with-service-name.yaml create mode 100644 tests/data/otel/default-configmap.yaml create mode 100644 tests/suite/test_otel.py diff --git a/tests/Makefile b/tests/Makefile index 3b955a6f04..8fe5f96035 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,7 +1,7 @@ SHELL = /bin/bash ROOT_DIR = $(shell git rev-parse --show-toplevel) CONTEXT = -PULL_POLICY = IfNotPresent +PULL_POLICY ?= IfNotPresent DEPLOYMENT_TYPE = deployment SERVICE = nodeport NODE_IP = diff --git a/tests/data/otel/configmap-with-all.yaml b/tests/data/otel/configmap-with-all.yaml new file mode 100644 index 0000000000..3f0dbecbac --- /dev/null +++ b/tests/data/otel/configmap-with-all.yaml @@ -0,0 +1,11 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" + otel-service-name: "nginx-ingress-controller:nginx" + otel-trace-in-http: "true" diff --git a/tests/data/otel/configmap-with-endpoint.yaml b/tests/data/otel/configmap-with-endpoint.yaml new file mode 100644 index 0000000000..fc462e2c79 --- /dev/null +++ b/tests/data/otel/configmap-with-endpoint.yaml @@ -0,0 +1,7 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" diff --git a/tests/data/otel/configmap-with-header.yaml b/tests/data/otel/configmap-with-header.yaml new file mode 100644 index 0000000000..868b7017f5 --- /dev/null +++ b/tests/data/otel/configmap-with-header.yaml @@ -0,0 +1,9 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" diff --git a/tests/data/otel/configmap-with-only-header-name.yaml b/tests/data/otel/configmap-with-only-header-name.yaml new file mode 100644 index 0000000000..edac4d37cc --- /dev/null +++ b/tests/data/otel/configmap-with-only-header-name.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-name: "x-otel-header" diff --git a/tests/data/otel/configmap-with-only-header-value.yaml b/tests/data/otel/configmap-with-only-header-value.yaml new file mode 100644 index 0000000000..d7f95030cb --- /dev/null +++ b/tests/data/otel/configmap-with-only-header-value.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-value: "otel-header-value" diff --git a/tests/data/otel/configmap-with-otel-trace.yaml b/tests/data/otel/configmap-with-otel-trace.yaml new file mode 100644 index 0000000000..7cb1e2a87f --- /dev/null +++ b/tests/data/otel/configmap-with-otel-trace.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-trace-in-http: "true" diff --git a/tests/data/otel/configmap-with-service-name.yaml b/tests/data/otel/configmap-with-service-name.yaml new file mode 100644 index 0000000000..29e29439cd --- /dev/null +++ b/tests/data/otel/configmap-with-service-name.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-service-name: "nginx-ingress-controller:nginx" diff --git a/tests/data/otel/default-configmap.yaml b/tests/data/otel/default-configmap.yaml new file mode 100644 index 0000000000..a6a6c812b5 --- /dev/null +++ b/tests/data/otel/default-configmap.yaml @@ -0,0 +1,6 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: diff --git a/tests/suite/test_otel.py b/tests/suite/test_otel.py new file mode 100644 index 0000000000..e48e503a11 --- /dev/null +++ b/tests/suite/test_otel.py @@ -0,0 +1,432 @@ +import pytest +from settings import TEST_DATA +from suite.utils.resources_utils import ( + get_nginx_template_conf, + replace_configmap_from_yaml, + wait_before_test, +) + +WAIT_TIME = 1 +cm_default = f"{TEST_DATA}/otel/default-configmap.yaml" +cm_endpoint = f"{TEST_DATA}/otel/configmap-with-endpoint.yaml" +cm_header = f"{TEST_DATA}/otel/configmap-with-header.yaml" +cm_header_only_name = f"{TEST_DATA}/otel/configmap-with-only-header-name.yaml" +cm_header_only_value = f"{TEST_DATA}/otel/configmap-with-only-header-value.yaml" +cm_service_name = f"{TEST_DATA}/otel/configmap-with-service-name.yaml" +cm_otel_trace = f"{TEST_DATA}/otel/configmap-with-otel-trace.yaml" +cm_all = f"{TEST_DATA}/otel/configmap-with-all.yaml" +otel_module = "modules/ngx_otel_module.so" +exporter = "otel.example.com:4317" +otel_exporter_header_name = "x-otel-header" +otel_exporter_header_value = "otel-header-value" +otel_service_name = "nginx-ingress-controller:nginx" +configmap_name = "nginx-config" + + +@pytest.mark.smoke +class TestOtel: + def extract_block(self, nginx_config, block_name): + """ + Extract a block of configuration from the nginx config file. + + :param nginx_config: The nginx config file content as a string. + :param block_name: The name of the block to extract. + :return: The extracted block as a string. + """ + start = nginx_config.find(block_name) + end = nginx_config.find("}", start) + 1 + return nginx_config[start:end] + + def test_otel_not_enabled( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts without otel configured in the `nginx-config` + 2. Ensure that the otel is not enabled in the nginx.conf + """ + + print("Step 1: apply default nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + + # Verify otel not present in nginx.conf + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + assert "otel" not in (nginx_config) + + def test_otel_endpoint( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `endpoint` is enabled in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_endpoint, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the endpoint is correctly configured") + assert f"endpoint {exporter};" in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + + def test_otel_header( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `header` is enabled in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_header, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the header is correctly configured") + assert f'header {otel_exporter_header_name} "{otel_exporter_header_value}";' in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_header_name_only( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `header` is not in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_header_only_name, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the header is not configured") + assert f"header" not in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_header_value_only( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `header` is not in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_header_only_value, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the header is not configured") + assert f"header" not in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_service_name( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `service-name` is enabled in the nginx.conf + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_service_name, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the service-name is correctly configured") + assert f"otel_service_name {otel_service_name}" in (nginx_config) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_trace( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that `otel_trace` is configured in the nginx.conf + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_otel_trace, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that otel_trace is configured") + assert "otel_trace on;" in (nginx_config) + + print("Step 5: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_all( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `endpoint` is enabled in the `otel_exporter` block. + 5. Ensure that the `header` is enabled in the `otel_exporter` block. + 6. Ensure that the `service-name` is enabled in the nginx.conf + 7. Ensure that `otel_trace` is configured in the nginx.conf + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_all, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = self.extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the endpoint is correctly configured") + assert f"endpoint {exporter};" in (exporter_block) + + print("Step 5: Ensure that the header is correctly configured") + assert f'header {otel_exporter_header_name} "{otel_exporter_header_value}";' in (exporter_block) + + print("Step 6: Ensure that the service-name is correctly configured") + assert f"otel_service_name {otel_service_name}" in (nginx_config) + + print("Step 7: Ensure that otel_trace is configured") + assert "otel_trace on;" in (nginx_config) + + print("Step 8: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py index 1916bf4992..4a49f2b06e 100644 --- a/tests/suite/utils/resources_utils.py +++ b/tests/suite/utils/resources_utils.py @@ -1011,7 +1011,7 @@ def clear_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace): ) -def get_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ic_pod_name=None) -> str: +def get_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ic_pod_name=None, print_log=True) -> str: """ Get contents of /etc/nginx/nginx.conf in the pod :param v1: CoreV1Api @@ -1022,7 +1022,7 @@ def get_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ic_pod_name=None) if ic_pod_name is None: ic_pod_name = get_first_pod_name(v1, ingress_namespace) file_path = "/etc/nginx/nginx.conf" - return get_file_contents(v1, file_path, ic_pod_name, ingress_namespace) + return get_file_contents(v1, file_path, ic_pod_name, ingress_namespace, print_log) def get_ingress_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ingress_name, pod_name, pod_namespace) -> str: From 3c8451d86c6b0f68fa94c95d9f61fa3cc960d292 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 14:24:08 +0100 Subject: [PATCH 33/55] build c-ares packages on UBI for Otel module --- .github/workflows/build-ubi-dependency.yml | 86 +++------------------- build/dependencies/Dockerfile.ubi | 33 --------- build/dependencies/Dockerfile.ubi8 | 10 +++ build/dependencies/Dockerfile.ubi9 | 10 +++ 4 files changed, 32 insertions(+), 107 deletions(-) delete mode 100644 build/dependencies/Dockerfile.ubi create mode 100644 build/dependencies/Dockerfile.ubi8 create mode 100644 build/dependencies/Dockerfile.ubi9 diff --git a/.github/workflows/build-ubi-dependency.yml b/.github/workflows/build-ubi-dependency.yml index 1f50a50132..d3dd346a3e 100644 --- a/.github/workflows/build-ubi-dependency.yml +++ b/.github/workflows/build-ubi-dependency.yml @@ -1,92 +1,36 @@ -name: Build UBI ppc64le Dependency +name: Build UBI c-ares Dependency on: push: branches: - main + - feat/add-otel-support paths: - - build/dependencies/Dockerfile.ubi + - build/dependencies/Dockerfile.ubi8 + - build/dependencies/Dockerfile.ubi9 workflow_dispatch: - inputs: - nginx_version: - type: string - description: "NGINX Version to build for" - required: false - force: - type: boolean - description: "Force rebuild" - required: false - default: false env: - IMAGE_NAME: ghcr.io/nginx/dependencies/nginx-ubi-ppc64le + IMAGE_NAME: ghcr.io/nginx/dependencies/nginx-ubi concurrency: - group: ${{ github.ref_name }}-ubi-ppc64le-build + group: ${{ github.ref_name }}-ubi-build cancel-in-progress: true permissions: contents: read jobs: - checks: - name: Check versions - runs-on: ubuntu-22.04 - permissions: - packages: read - contents: read - strategy: - fail-fast: false - outputs: - nginx_version: ${{ steps.var.outputs.nginx_version }} - njs_version: ${{ steps.var.outputs.njs_version }} - target_exists: ${{ steps.var.outputs.target_image_exists }} - steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Output Variables - id: var - run: | - if [ -n "${{ inputs.nginx_version }}" ]; then - nginx_v=${{ inputs.nginx_version }} - else - nginx_v=$(grep -m1 'FROM nginx:' Outputs -------------------------------" - echo "NJS_VERSION=$njs" - echo "nginx_version=${nginx_v}" - echo "njs_version=${njs}" - echo "target_image_exists=${target_image_exists}" - echo "nginx_version=${nginx_v}" >> $GITHUB_OUTPUT - echo "njs_version=${njs}" >> $GITHUB_OUTPUT - echo "target_image_exists=${target_image_exists}" >> $GITHUB_OUTPUT - build-binaries: name: Build Binary Container Image - if: ${{ needs.checks.outputs.target_exists != 'true' || inputs.force }} - needs: checks runs-on: ubuntu-22.04 permissions: packages: write contents: read strategy: fail-fast: false + matrix: + tag: ["ubi8", "ubi9"] steps: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -113,28 +57,22 @@ jobs: images: | name=${{ env.IMAGE_NAME }},enable=true tags: | - type=raw,value=nginx-${{ needs.checks.outputs.nginx_version }},enable=true + type=raw,value=${{ matrix.tag }},enable=true env: DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - name: Build and push uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0 with: - file: ./build/dependencies/Dockerfile.ubi + file: ./build/dependencies/Dockerfile.${{ matrix.tag }} context: "." pull: true - push: true - # build multi-arch so that it can be mounted from any image - # even though only ppc64le will contain binaries platforms: "linux/amd64,linux/arm64" tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} - cache-from: type=gha,scope=nginx-ubi-ppc64le - cache-to: type=gha,scope=nginx-ubi-ppc64le,mode=max + cache-from: type=gha,scope=nginx-${{ matrix.tag }} + cache-to: type=gha,scope=nginx-${{ matrix.tag }},mode=max target: final sbom: false provenance: mode=max - build-args: | - NGINX=${{ needs.checks.outputs.nginx_version }} - NJS=${{ needs.checks.outputs.njs_version }} diff --git a/build/dependencies/Dockerfile.ubi b/build/dependencies/Dockerfile.ubi deleted file mode 100644 index ecc5862f4d..0000000000 --- a/build/dependencies/Dockerfile.ubi +++ /dev/null @@ -1,33 +0,0 @@ -# syntax=docker/dockerfile:1.8 -FROM nginx:1.27.4@sha256:09369da6b10306312cd908661320086bf87fbae1b6b0c49a1f50ba531fef2eab AS nginx - -FROM redhat/ubi9:9.5@sha256:d07a5e080b8a9b3624d3c9cfbfada9a6baacd8e6d4065118f0e80c71ad518044 AS rpm-build -ARG NGINX -ARG NJS -ENV NGINX_VERSION=${NGINX} -ENV NJS_VERSION=${NJS} -RUN mkdir -p /nginx/; \ - # only build for ppc64le but make multiarch image for mounting - [ $(uname -p) = x86_64 ] && exit 0; \ - [ $(uname -p) = aarch64 ] && exit 0; \ - rpm --import https://nginx.org/keys/nginx_signing.key \ - && MINOR_VERSION=$(echo ${NGINX_VERSION} | cut -d '.' -f 2) \ - && if [ $(( $MINOR_VERSION % 2)) -eq 0 ]; then echo mainline=""; else mainline="mainline/"; fi \ - && printf "%s\n" "[nginx]" "name=nginx src repo" \ - "baseurl=https://nginx.org/packages/${mainline}centos/9/SRPMS" \ - "gpgcheck=1" "enabled=1" "module_hotfixes=true" >> /etc/yum.repos.d/nginx.repo \ - && dnf install rpm-build gcc make dnf-plugins-core which -y \ - && dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ - && nginxPackages=" \ - nginx-${NGINX_VERSION} \ - nginx-module-xslt-${NGINX_VERSION} \ - nginx-module-image-filter-${NGINX_VERSION} \ - nginx-module-njs-${NGINX_VERSION}+${NJS_VERSION} \ - " \ - && dnf download --source ${nginxPackages} \ - && dnf builddep -y --srpm nginx*.rpm \ - && rpmbuild --rebuild --nodebuginfo nginx*.rpm \ - && cp /root/rpmbuild/RPMS/$(arch)/* /nginx/ - -FROM scratch AS final -COPY --link --from=rpm-build /nginx / diff --git a/build/dependencies/Dockerfile.ubi8 b/build/dependencies/Dockerfile.ubi8 new file mode 100644 index 0000000000..3cb3dc7c01 --- /dev/null +++ b/build/dependencies/Dockerfile.ubi8 @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1.8 +FROM redhat/ubi8@sha256:244e9858f9d8a2792a3dceb850b4fa8fdbd67babebfde42587bfa919d5d1ecef AS rpm-build +RUN mkdir -p /rpms/ \ + && dnf install rpm-build gcc make cmake -y \ + && rpmbuild --rebuild --nodebuginfo https://mirror.stream.centos.org/9-stream/BaseOS/source/tree/Packages/c-ares-1.19.1-1.el9.src.rpm \ + && cp /root/rpmbuild/RPMS/$(arch)/* /rpms/ \ + && rm -rf /rpms/*devel* + +FROM scratch AS final +COPY --link --from=rpm-build /rpms / diff --git a/build/dependencies/Dockerfile.ubi9 b/build/dependencies/Dockerfile.ubi9 new file mode 100644 index 0000000000..25ef55a9fb --- /dev/null +++ b/build/dependencies/Dockerfile.ubi9 @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1.8 +FROM redhat/ubi9:9.5@sha256:d07a5e080b8a9b3624d3c9cfbfada9a6baacd8e6d4065118f0e80c71ad518044 AS rpm-build +RUN mkdir -p /rpms/ \ + && dnf install rpm-build gcc make cmake -y \ + && rpmbuild --rebuild --nodebuginfo https://mirror.stream.centos.org/9-stream/BaseOS/source/tree/Packages/c-ares-1.19.1-1.el9.src.rpm \ + && cp /root/rpmbuild/RPMS/$(arch)/* /rpms/ \ + && rm -rf /rpms/*devel* + +FROM scratch AS final +COPY --link --from=rpm-build /rpms / From adc844f87383fe443db530152bd02f1943f71f44 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 14:36:22 +0100 Subject: [PATCH 34/55] push image --- .github/workflows/build-ubi-dependency.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-ubi-dependency.yml b/.github/workflows/build-ubi-dependency.yml index d3dd346a3e..86acf88cdd 100644 --- a/.github/workflows/build-ubi-dependency.yml +++ b/.github/workflows/build-ubi-dependency.yml @@ -67,6 +67,7 @@ jobs: file: ./build/dependencies/Dockerfile.${{ matrix.tag }} context: "." pull: true + push: true platforms: "linux/amd64,linux/arm64" tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} From c9635872d0fc81be205937c19d1be05c84a3bef8 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 14:41:58 +0100 Subject: [PATCH 35/55] trigger build by changing CI file --- .github/workflows/build-ubi-dependency.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-ubi-dependency.yml b/.github/workflows/build-ubi-dependency.yml index 86acf88cdd..404e153ce4 100644 --- a/.github/workflows/build-ubi-dependency.yml +++ b/.github/workflows/build-ubi-dependency.yml @@ -8,6 +8,7 @@ on: paths: - build/dependencies/Dockerfile.ubi8 - build/dependencies/Dockerfile.ubi9 + - .github/workflows/build-ubi-dependency.yml workflow_dispatch: env: From 1d80cfd699dea4b5620dd064ff45c184458137f1 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 16:34:52 +0100 Subject: [PATCH 36/55] add c-ares to all ubi based images --- build/Dockerfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 2e54241801..0fca643ab6 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -12,6 +12,8 @@ ARG PACKAGE_REPO=pkgs.nginx.com ############################################# Base images containing libs for FIPS ############################################# FROM ghcr.io/nginx/dependencies/nginx-ubi-ppc64le:nginx-1.27.4@sha256:fff4dde599b89cb22e5cea5d8cfba8c47bcedaa8e6fa549f5fe74a89c733aa2f AS ubi-ppc64le +FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi8@sha256:7ddf7a566cb0c44e21b3652704e49c7f43e4d18dd875e1bbd4c4ec19d0b70989 AS ubi8-packages +FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi9@sha256:a5dc7c5fef28271ecb651801785b18702e57b5ecd7cc8aa8867982c193d27d5b AS ubi9-packages FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.19@sha256:2a7f8451110b588b733e4cb8727a48153057b1debac5c78ef8a539ff63712fa1 AS alpine-fips-3.19 FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.21@sha256:5221dec2e33436f2586c743c7aa3ef4626c0ec54184dc3364d101036d4f4a060 AS alpine-fips-3.21 FROM redhat/ubi9-minimal:9.5@sha256:a50731d3397a4ee28583f1699842183d4d24fadcc565c4688487af9ee4e13a44 AS ubi-minimal @@ -342,23 +344,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_signing.key \ --mount=type=bind,from=nginx-files,src=ubi-setup.sh,target=/usr/local/bin/ubi-setup.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ - --mount=type=bind,from=ubi-ppc64le,src=/,target=/ubi-bin/ \ - ubi-setup.sh; \ - if [ $(uname -p) = ppc64le ] || [ $(uname -p) = s390x ]; then \ - rpm -qa --queryformat "%{NAME}\n" | sort > pkgs-installed \ - && microdnf --nodocs --setopt=install_weak_deps=0 install -y diffutils dnf \ - && rpm -qa --queryformat "%{NAME}\n" | sort > pkgs-new \ - && dnf install -y /ubi-bin/*.rpm \ - && dnf -q repoquery --resolve --requires --recursive --whatrequires nginx --queryformat "%{NAME}" > pkgs-nginx \ - && dnf --setopt=protected_packages= remove -y $(comm -13 pkgs-installed pkgs-new | comm -13 pkgs-nginx -) \ - && rm pkgs-installed pkgs-new pkgs-nginx; \ - else \ - printf "%s\n" "[nginx]" "name=nginx repo" \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ + ubi-setup.sh \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ + && printf "%s\n" "[nginx]" "name=nginx repo" \ "baseurl=https://nginx.org/packages/mainline/centos/9/\$basearch/" \ "gpgcheck=1" "enabled=1" "module_hotfixes=true" > /etc/yum.repos.d/nginx.repo \ && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel nginx-module-image-filter nginx-module-xslt \ - && rm /etc/yum.repos.d/nginx.repo; \ - fi \ + && rm /etc/yum.repos.d/nginx.repo \ && ubi-clean.sh @@ -376,8 +369,10 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=ubi-setup.sh,target=/usr/local/bin/ubi-setup.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ubi-setup.sh \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && ubi-clean.sh @@ -400,8 +395,10 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-dos.sh,target=/usr/local/bin/nap-dos.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ && microdnf --nodocs install -y ca-certificates shadow-utils subscription-manager \ && if [ "${NGINX_AGENT}" = "true" ]; then microdnf --nodocs install -y nginx-agent; fi \ @@ -441,8 +438,10 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-waf.sh,target=/usr/local/bin/nap-waf.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ && microdnf --nodocs install -y ca-certificates shadow-utils subscription-manager \ && if [ "${NGINX_AGENT}" = "true" ]; then microdnf --nodocs install -y nginx-agent; fi \ @@ -475,8 +474,10 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-waf.sh,target=/usr/local/bin/nap-waf.sh \ --mount=type=bind,from=nginx-files,src=agent.sh,target=/usr/local/bin/agent.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi8-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ cp /tmp/app-protect-8.repo /etc/yum.repos.d/app-protect-8.repo; \ fi \ @@ -521,8 +522,10 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-waf.sh,target=/usr/local/bin/nap-waf.sh \ --mount=type=bind,from=nginx-files,src=agent.sh,target=/usr/local/bin/agent.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi8-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ cp /tmp/app-protect-8.repo /etc/yum.repos.d/app-protect-8.repo; \ fi \ From 12413731f3c2589b7750148f0644c10b81cd1912 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 16:36:29 +0100 Subject: [PATCH 37/55] remove testing branch from dependency dockerfile --- .github/workflows/build-ubi-dependency.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-ubi-dependency.yml b/.github/workflows/build-ubi-dependency.yml index 404e153ce4..ff6b35ec62 100644 --- a/.github/workflows/build-ubi-dependency.yml +++ b/.github/workflows/build-ubi-dependency.yml @@ -4,7 +4,6 @@ on: push: branches: - main - - feat/add-otel-support paths: - build/dependencies/Dockerfile.ubi8 - build/dependencies/Dockerfile.ubi9 From 39db63269f1c0a1c6e48c0972db7abeba7d5c643 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 16:44:21 +0100 Subject: [PATCH 38/55] remove duplicated package installation --- build/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 0fca643ab6..53d38a18c8 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -395,10 +395,8 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-dos.sh,target=/usr/local/bin/nap-dos.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ - --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ - && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ && microdnf --nodocs install -y ca-certificates shadow-utils subscription-manager \ && if [ "${NGINX_AGENT}" = "true" ]; then microdnf --nodocs install -y nginx-agent; fi \ @@ -438,10 +436,8 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-waf.sh,target=/usr/local/bin/nap-waf.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ - --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ - && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ && microdnf --nodocs install -y ca-certificates shadow-utils subscription-manager \ && if [ "${NGINX_AGENT}" = "true" ]; then microdnf --nodocs install -y nginx-agent; fi \ From a4de5c651bf517c32b94a726a151711a41811180 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Fri, 16 May 2025 16:50:07 +0100 Subject: [PATCH 39/55] add otel tests to all images --- .github/data/matrix-smoke-nap.json | 10 +++++----- .github/data/matrix-smoke-oss.json | 28 ++++++++++++++-------------- .github/data/matrix-smoke-plus.json | 6 +++--- pyproject.toml | 1 + tests/suite/test_otel.py | 2 +- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/.github/data/matrix-smoke-nap.json b/.github/data/matrix-smoke-nap.json index cd2bcba29a..2d77824834 100644 --- a/.github/data/matrix-smoke-nap.json +++ b/.github/data/matrix-smoke-nap.json @@ -5,7 +5,7 @@ "image": "ubi-8-plus-nap", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_policies_allow", + "marker": "appprotect_waf_policies_allow or otel", "platforms": "linux/amd64" }, { @@ -21,7 +21,7 @@ "image": "alpine-plus-nap-fips", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_policies_grpc", + "marker": "appprotect_waf_policies_grpc or otel", "platforms": "linux/amd64" }, { @@ -29,7 +29,7 @@ "image": "debian-plus-nap", "type": "plus", "nap_modules": "waf", - "marker": "'appprotect_watch or appprotect_batch or appprotect_integration or appprotect_waf_policies_vsr'", + "marker": "'appprotect_watch or appprotect_batch or appprotect_integration or appprotect_waf_policies_vsr or otel'", "platforms": "linux/amd64" }, { @@ -37,7 +37,7 @@ "image": "debian-plus-nap-v5", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_v5", + "marker": "appprotect_waf_v5 or otel", "platforms": "linux/amd64" }, { @@ -61,7 +61,7 @@ "image": "ubi-9-plus-nap", "type": "plus", "nap_modules": "dos", - "marker": "dos_learning", + "marker": "dos_learning or otel", "platforms": "linux/amd64" }, { diff --git a/.github/data/matrix-smoke-oss.json b/.github/data/matrix-smoke-oss.json index a15b9b8937..cca3b5189e 100644 --- a/.github/data/matrix-smoke-oss.json +++ b/.github/data/matrix-smoke-oss.json @@ -5,77 +5,77 @@ "image": "debian", "type": "oss", "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "ingresses 2/2", "image": "debian", "type": "oss", "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR 1/3", "image": "alpine", "type": "oss", "marker": "'vsr and not vsr_upstream and not vsr_grpc and not vsr_status and not vsr_canary and not vsr_routing and not vsr_api and not vsr_redirects and not vsr_rewrite and not vsr_canned and not vsr_basic'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR 2/3", "image": "alpine", "type": "oss", "marker": "'vsr_basic or vsr_canned or vsr_rewrite or vsr_redirects or vsr_upstream'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR 3/3", "image": "alpine", "type": "oss", "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "policies 1/2", "image": "alpine", "type": "oss", "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "policies 2/2", "image": "alpine", "type": "oss", - "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls or otel'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 1/3", "image": "debian", "type": "oss", "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 2/3", "image": "debian", "type": "oss", "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_api or vs_backup or vs_use_cluster_ip or vs_canary or vs_upstream or vs_config_map'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 3/3", "image": "debian", "type": "oss", - "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager or otel'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "TS", "image": "ubi", "type": "oss", - "marker": "ts", - "platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "marker": "ts or otel", + "platforms": "linux/arm64, linux/amd64" } ], "k8s": [] diff --git a/.github/data/matrix-smoke-plus.json b/.github/data/matrix-smoke-plus.json index 247ad5370a..ddad998424 100644 --- a/.github/data/matrix-smoke-plus.json +++ b/.github/data/matrix-smoke-plus.json @@ -25,7 +25,7 @@ "label": "TS", "image": "debian-plus", "type": "plus", - "marker": "ts", + "marker": "ts or otel", "platforms": "linux/arm64, linux/amd64" }, { @@ -60,7 +60,7 @@ "label": "VSR 3/3", "image": "alpine-plus", "type": "plus", - "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc'", + "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc or otel'", "platforms": "linux/arm64, linux/amd64" }, { @@ -74,7 +74,7 @@ "label": "policies 2/3", "image": "ubi-9-plus", "type": "plus", - "marker": "'policies_ac or policies_jwt or policies_mtls'", + "marker": "'policies_ac or policies_jwt or policies_mtls or otel'", "platforms": "linux/arm64, linux/amd64, linux/s390x" }, { diff --git a/pyproject.toml b/pyproject.toml index a2bdec0dcb..cfd6f488bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ markers = [ "ingresses", "multi_ns", "oidc", + "otel", "policies", "policies_rl", "policies_jwt", diff --git a/tests/suite/test_otel.py b/tests/suite/test_otel.py index e48e503a11..9562ec39ce 100644 --- a/tests/suite/test_otel.py +++ b/tests/suite/test_otel.py @@ -23,7 +23,7 @@ configmap_name = "nginx-config" -@pytest.mark.smoke +@pytest.mark.otel class TestOtel: def extract_block(self, nginx_config, block_name): """ From 15b8fbbe1e21e357aac5d9bd1d05c79ff9a572c4 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 19 May 2025 10:48:47 +0100 Subject: [PATCH 40/55] move extract_block() to resource_utils package --- tests/suite/test_otel.py | 26 ++++++++------------------ tests/suite/utils/resources_utils.py | 13 +++++++++++++ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/tests/suite/test_otel.py b/tests/suite/test_otel.py index 9562ec39ce..9f7f2fe793 100644 --- a/tests/suite/test_otel.py +++ b/tests/suite/test_otel.py @@ -1,6 +1,7 @@ import pytest from settings import TEST_DATA from suite.utils.resources_utils import ( + extract_block, get_nginx_template_conf, replace_configmap_from_yaml, wait_before_test, @@ -25,17 +26,6 @@ @pytest.mark.otel class TestOtel: - def extract_block(self, nginx_config, block_name): - """ - Extract a block of configuration from the nginx config file. - - :param nginx_config: The nginx config file content as a string. - :param block_name: The name of the block to extract. - :return: The extracted block as a string. - """ - start = nginx_config.find(block_name) - end = nginx_config.find("}", start) + 1 - return nginx_config[start:end] def test_otel_not_enabled( self, @@ -96,7 +86,7 @@ def test_otel_endpoint( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) @@ -147,7 +137,7 @@ def test_otel_header( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) @@ -199,7 +189,7 @@ def test_otel_header_name_only( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) @@ -251,7 +241,7 @@ def test_otel_header_value_only( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) @@ -303,7 +293,7 @@ def test_otel_service_name( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) @@ -354,7 +344,7 @@ def test_otel_trace( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) @@ -405,7 +395,7 @@ def test_otel_all( print("Step 2: Ensure that the otel module is loaded") assert otel_module in (nginx_config) - exporter_block = self.extract_block(nginx_config, "otel_exporter") + exporter_block = extract_block(nginx_config, "otel_exporter") print("Step 3: Ensure that the otel_exporter is enabled") assert "otel_exporter" in (exporter_block) diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py index 4a49f2b06e..9a25cc0aea 100644 --- a/tests/suite/utils/resources_utils.py +++ b/tests/suite/utils/resources_utils.py @@ -1070,6 +1070,19 @@ def get_ts_nginx_template_conf(v1: CoreV1Api, resource_namespace, resource_name, return get_file_contents(v1, file_path, pod_name, pod_namespace) +def extract_block(nginx_config, block_name): + """ + Extract a block of configuration from the nginx config file. + + :param nginx_config: The nginx config file content as a string. + :param block_name: The name of the block to extract. + :return: The extracted block as a string. + """ + start = nginx_config.find(block_name) + end = nginx_config.find("}", start) + 1 + return nginx_config[start:end] + + def create_example_app(kube_apis, app_type, namespace) -> None: """ Create a backend application. From c6fa5b851ec4a7bc37d4243a51ab4ced2bcf163b Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 19 May 2025 10:49:30 +0100 Subject: [PATCH 41/55] add test to ensure otel is disabled if endpoint is missing --- .../configmap-with-all-except-endpoint.yaml | 10 ++++ tests/suite/test_otel.py | 51 +++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 tests/data/otel/configmap-with-all-except-endpoint.yaml diff --git a/tests/data/otel/configmap-with-all-except-endpoint.yaml b/tests/data/otel/configmap-with-all-except-endpoint.yaml new file mode 100644 index 0000000000..9940afb925 --- /dev/null +++ b/tests/data/otel/configmap-with-all-except-endpoint.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" + otel-service-name: "nginx-ingress-controller:nginx" + otel-trace-in-http: "true" diff --git a/tests/suite/test_otel.py b/tests/suite/test_otel.py index 9f7f2fe793..153527ca3a 100644 --- a/tests/suite/test_otel.py +++ b/tests/suite/test_otel.py @@ -16,6 +16,7 @@ cm_service_name = f"{TEST_DATA}/otel/configmap-with-service-name.yaml" cm_otel_trace = f"{TEST_DATA}/otel/configmap-with-otel-trace.yaml" cm_all = f"{TEST_DATA}/otel/configmap-with-all.yaml" +cm_all_except_endpoint = f"{TEST_DATA}/otel/configmap-with-all-except-endpoint.yaml" otel_module = "modules/ngx_otel_module.so" exporter = "otel.example.com:4317" otel_exporter_header_name = "x-otel-header" @@ -420,3 +421,53 @@ def test_otel_all( cm_default, ) wait_before_test(WAIT_TIME) + + def test_otel_all_except_endpoint( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with all otel configuration except endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is not in the nginx.conf + 3. Ensure that the `otel_exporter` is not in the nginx.conf + 4. Ensure that the `service-name` is not in the nginx.conf + 5. Ensure that `otel_trace` is not in the nginx.conf + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_all_except_endpoint, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is not loaded") + assert otel_module not in (nginx_config) + + print("Step 3: Ensure that the otel_exporter is not enabled") + assert "otel_exporter" not in (nginx_config) + + print("Step 4: Ensure that the service-name is not correctly configured") + assert f"otel_service_name {otel_service_name}" not in (nginx_config) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace on;" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) From 31412e6db49d5ee01f7031a70e51eb4f5eb04e09 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Mon, 19 May 2025 09:34:48 +0100 Subject: [PATCH 42/55] update docs Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- .../configuration/global-configuration/configmap-resource.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/site/content/configuration/global-configuration/configmap-resource.md b/site/content/configuration/global-configuration/configmap-resource.md index 724f6bb05e..715792d0eb 100644 --- a/site/content/configuration/global-configuration/configmap-resource.md +++ b/site/content/configuration/global-configuration/configmap-resource.md @@ -230,6 +230,11 @@ If you encounter the error `error [emerg] 13#13: "zone_sync" directive is duplic {{}} |ConfigMap Key | Description | Default | Example | | ---| ---| ---| --- | +|*otel-exporter-endpoint* | OTLP/gRPC endpoint that will accept [OpenTelemetry](https://opentelemetry.io) data. Set `otel-trace-in-http` to *True* to enable OpenTelemetry at global level. Example: *"https://otel-collector:4317"*. Note: requires the Ingress Controller image with OpenTelemetry module. | | | +|*otel-exporter-header-name* | The name of a custom HTTP header to add to telemetry export request. Example: *"X-custom-header"*. `otel-exporter-endpoint` and `otel-exporter-header-value` required. | | | +|*otel-exporter-header-value* | The value of a custom HTTP header to add to telemetry export request.Example: *"custom-value"*. `otel-exporter-endpoint` and `otel-exporter-header-name` required. | | | +|*otel-service-name* | Sets the `service.name` attribute of the OTel resource. Example: *"nginx-ingress-controller:nginx"*. `otel-exporter-endpoint` required. | | | +| *otel-trace-in-http* | Enables [OpenTelemetry](https://opentelemetry.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Set this to *False* to enable OpenTelemetry for individual routes with snippets. `otel-exporter-endpoint` required. | *False* | | |*opentracing* | Removed in v5.0.0. Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs]({{< relref "/installation/integrations/opentracing.md" >}}) for more information. | *False* | | |*opentracing-tracer* | Removed in v5.0.0. Sets the path to the vendor tracer binary plugin. | N/A | | |*opentracing-tracer-config* | Removed in v5.0.0. Sets the tracer configuration in JSON format. | N/A | | From a4d45e4e51e83a564e3379dd27248fd6e87f0002 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 19 May 2025 11:06:40 +0100 Subject: [PATCH 43/55] add single qoutes to NAP test markers --- .github/data/matrix-smoke-nap.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/data/matrix-smoke-nap.json b/.github/data/matrix-smoke-nap.json index 2d77824834..e0c8c1ac3b 100644 --- a/.github/data/matrix-smoke-nap.json +++ b/.github/data/matrix-smoke-nap.json @@ -5,7 +5,7 @@ "image": "ubi-8-plus-nap", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_policies_allow or otel", + "marker": "'appprotect_waf_policies_allow or otel'", "platforms": "linux/amd64" }, { @@ -21,7 +21,7 @@ "image": "alpine-plus-nap-fips", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_policies_grpc or otel", + "marker": "'appprotect_waf_policies_grpc or otel'", "platforms": "linux/amd64" }, { @@ -37,7 +37,7 @@ "image": "debian-plus-nap-v5", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_v5 or otel", + "marker": "'appprotect_waf_v5 or otel'", "platforms": "linux/amd64" }, { @@ -61,7 +61,7 @@ "image": "ubi-9-plus-nap", "type": "plus", "nap_modules": "dos", - "marker": "dos_learning or otel", + "marker": "'dos_learning or otel'", "platforms": "linux/amd64" }, { From 55b554748ac49aaf00dc59dcbca8c42ac481b3d7 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Mon, 19 May 2025 11:15:03 +0100 Subject: [PATCH 44/55] add single qoutes to all otel test markers --- .github/data/matrix-smoke-oss.json | 2 +- .github/data/matrix-smoke-plus.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/data/matrix-smoke-oss.json b/.github/data/matrix-smoke-oss.json index cca3b5189e..6c402dc04f 100644 --- a/.github/data/matrix-smoke-oss.json +++ b/.github/data/matrix-smoke-oss.json @@ -74,7 +74,7 @@ "label": "TS", "image": "ubi", "type": "oss", - "marker": "ts or otel", + "marker": "'ts or otel'", "platforms": "linux/arm64, linux/amd64" } ], diff --git a/.github/data/matrix-smoke-plus.json b/.github/data/matrix-smoke-plus.json index ddad998424..4bded89736 100644 --- a/.github/data/matrix-smoke-plus.json +++ b/.github/data/matrix-smoke-plus.json @@ -25,7 +25,7 @@ "label": "TS", "image": "debian-plus", "type": "plus", - "marker": "ts or otel", + "marker": "'ts or otel'", "platforms": "linux/arm64, linux/amd64" }, { From 89b15799358b4f7a4ad7caeac6652d2ef2575c71 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 20 May 2025 09:20:50 +0100 Subject: [PATCH 45/55] add example --- examples/shared-examples/otel/README.md | 82 +++++++++++++++++++ .../shared-examples/otel/nginx-config.yaml | 11 +++ 2 files changed, 93 insertions(+) create mode 100644 examples/shared-examples/otel/README.md create mode 100644 examples/shared-examples/otel/nginx-config.yaml diff --git a/examples/shared-examples/otel/README.md b/examples/shared-examples/otel/README.md new file mode 100644 index 0000000000..6ef1214658 --- /dev/null +++ b/examples/shared-examples/otel/README.md @@ -0,0 +1,82 @@ +# Learn how to use OpenTelemetry with F5 NGINX Ingress Controller + +NGINX Ingress Controller supports [OpenTelemetry](https://opentelemetry.io/) with the NGINX module [ngx_otel_module](https://nginx.org/en/docs/ngx_otel_module.html). + +## Prerequisites + +1. Use a NGINX Ingress Controller image that contains OpenTelemetry. + + - All NGINX Ingress Controller v5.1 images or later will contain support for `ngx_otel_module`. + - Alternatively, you follow [Build NGINX Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/installation/build-nginx-ingress-controller/) using `debian-image` (or `alpine-image` or `ubi-image`) for NGINX or `debian-image-plus` (or `alpine-image-plus`or `ubi-image-plus`) for NGINX Plus. + +1. Enable snippets annotations by setting the [`enable-snippets`](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-enable-snippets) command-line argument to true. + +1. Load the OpenTelemetry module. + + You need to load the module using the following ConfigMap key: + + - `otel-exporter-endpoint`: sets the endpoint to export your OpenTelemetry traces to. + + The following example shows how to use this to export data to an OpenTelemetry collector running in your cluster: + + ```yaml + otel-exporter-endpoint: "http://otel-collector.default.svc.cluster.local:4317" + ``` + +## Enable OpenTelemetry globally + +To enable OpenTelemetry globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `otel-trace-in-http` ConfigMap key to `true`: + +```yaml +otel-trace-in-http: "true" +``` + +## Enable or disable OpenTelemetry per Ingress resource + +You can use annotations to enable or disable OpenTelemetry for a specific Ingress resource. As mentioned in the prerequisites section, `otel-exporter-endpoint` must be configured. + +Consider the following two cases: + +### OpenTelemetry is globally disabled + +1. To enable OpenTelemetry for a specific Ingress resource, use the server snippet annotation: + + ```yaml + nginx.org/server-snippets: | + otel_trace on; + ``` + +1. To enable OpenTelemetry for specific paths: + + - You need to use [Mergeable Ingress resources](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration) + - You need to use the location snippets annotation to enable OpenTelemetry for the paths of a specific Minion Ingress resource: + + ```yaml + nginx.org/location-snippets: | + otel_trace on; + ``` + +### OpenTelemetry is globally enabled + +1. To disable OpenTelemetry for a specific Ingress resource, use the server snippet annotation: + + ```yaml + nginx.org/server-snippets: | + otel_trace off; + ``` + +1. To disable OpenTelemetry for specific paths: + + - You need to use [Mergeable Ingress resources](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration) + - You need to use the location snippets annotation to disable OpenTelemetry for the paths of a specific Minion Ingress resource: + + ```yaml + nginx.org/location-snippets: | + otel_trace off; + ``` + +## Customize OpenTelemetry + +You can customize OpenTelemetry through the supported [OpenTelemetry module directives](https://nginx.org/en/docs/ngx_otel_module.html). Use the `location-snippets` ConfigMap keys or annotations to insert those directives into the generated NGINX configuration. + +> Note: At present, the additional directives in the `otel_exporter` block cannot be modified with snippets. diff --git a/examples/shared-examples/otel/nginx-config.yaml b/examples/shared-examples/otel/nginx-config.yaml new file mode 100644 index 0000000000..8b3cb5c65b --- /dev/null +++ b/examples/shared-examples/otel/nginx-config.yaml @@ -0,0 +1,11 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-service-name: "nginx-ingress-controller:nginx" + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" + # otel-trace-in-http: "true" From ab7fcf56a6882060509a384d61adaa3ba1e091d1 Mon Sep 17 00:00:00 2001 From: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Date: Tue, 20 May 2025 11:30:35 +0100 Subject: [PATCH 46/55] allow partially valid otel config, add validation and unit tests Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> Signed-off-by: Haywood Shannon <5781935+haywoodsh@users.noreply.github.com> --- internal/configs/configmaps.go | 24 +++++- internal/configs/configmaps_test.go | 113 +++++++++++++++++++++++----- 2 files changed, 116 insertions(+), 21 deletions(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 54c7cfc39c..d35d66de8b 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/nginx/kubernetes-ingress/internal/validation" + "github.com/nginx/kubernetes-ingress/internal/configs/commonhelpers" v1 "k8s.io/api/core/v1" @@ -16,7 +18,7 @@ import ( "github.com/nginx/kubernetes-ingress/internal/configs/version1" nl "github.com/nginx/kubernetes-ingress/internal/logger" - "github.com/nginx/kubernetes-ingress/internal/validation" + k8s_validation "k8s.io/apimachinery/pkg/util/validation" ) const ( @@ -747,6 +749,8 @@ func parseConfigMapZoneSync(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams *Confi //nolint:gocyclo func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams *ConfigParams, eventLog record.EventRecorder) (*ConfigParams, error) { + otelValid := true + if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) if otelExporterEndpoint != "" { @@ -757,7 +761,15 @@ func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams * if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) if otelExporterHeaderName != "" { - cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName + errorMessages := k8s_validation.IsHTTPHeaderName(otelExporterHeaderName) + if len(errorMessages) > 0 { + errorText := fmt.Sprintf("ConfigMap %s/%s: invalid value for 'otel-exporter-header-name': %q, %v", cfgm.GetNamespace(), cfgm.GetName(), otelExporterHeaderName, errorMessages) + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + otelValid = false + } else { + cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName + } } } @@ -775,8 +787,6 @@ func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams * } } - otelValid := true - if otelTraceInHTTP, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-trace-in-http", cfgm); exists { if err != nil { nl.Error(l, err) @@ -788,6 +798,8 @@ func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams * if (cfgParams.MainOtelExporterHeaderName != "" && cfgParams.MainOtelExporterHeaderValue == "") || (cfgParams.MainOtelExporterHeaderName == "" && cfgParams.MainOtelExporterHeaderValue != "") { + cfgParams.MainOtelExporterHeaderName = "" + cfgParams.MainOtelExporterHeaderValue = "" errorText := "Both 'otel-exporter-header-name' and 'otel-exporter-header-value' must be set or neither" nl.Error(l, errorText) eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) @@ -808,6 +820,10 @@ func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams * eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) otelValid = false cfgParams.MainOtelTraceInHTTP = false + cfgParams.MainOtelExporterHeaderName = "" + cfgParams.MainOtelExporterHeaderValue = "" + cfgParams.MainOtelServiceName = "" + cfgParams.MainOtelTraceInHTTP = false } if !otelValid { diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index 1559ede49d..397e6da9ca 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -1440,7 +1440,7 @@ func TestOpenTelemetryConfigurationSuccess(t *testing.T) { hasAppProtect := false hasAppProtectDos := false hasTLSPassthrough := false - const expectedConfigOk = true + expectedConfigOk := true for _, test := range tests { t.Run(test.msg, func(t *testing.T) { @@ -1474,19 +1474,44 @@ func TestOpenTelemetryConfigurationSuccess(t *testing.T) { func TestOpenTelemetryConfigurationInvalid(t *testing.T) { t.Parallel() tests := []struct { - configMap *v1.ConfigMap - expectedLoadModule bool - msg string + configMap *v1.ConfigMap + expectedLoadModule bool + expectedExporterEndpoint string + expectedExporterHeaderName string + expectedExporterHeaderValue string + expectedServiceName string + expectedTraceInHTTP bool + msg string }{ { configMap: &v1.ConfigMap{ Data: map[string]string{ - "otel-exporter-endpoint": " ", + "otel-exporter-endpoint": "", "otel-service-name": "nginx-ingress-controller:nginx", }, }, - expectedLoadModule: false, - msg: "empty endpoint, service name set", + expectedLoadModule: false, + expectedExporterEndpoint: "", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "", + expectedTraceInHTTP: false, + msg: "invalid, endpoint missing, service name set", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-header-name": "X-Custom-Header", + "otel-exporter-header-value": "custom-value", + }, + }, + expectedLoadModule: false, + expectedExporterEndpoint: "", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "", + expectedTraceInHTTP: false, + msg: "invalid, endpoint missing, header name and value set", }, { configMap: &v1.ConfigMap{ @@ -1496,8 +1521,13 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { "otel-service-name": "nginx-ingress-controller:nginx", }, }, - expectedLoadModule: true, - msg: "endpoint set, header name only", + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, header value missing", }, { configMap: &v1.ConfigMap{ @@ -1507,18 +1537,30 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { "otel-service-name": "nginx-ingress-controller:nginx", }, }, - expectedLoadModule: true, - msg: "endpoint set, header value only", + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, header name missing", }, { configMap: &v1.ConfigMap{ Data: map[string]string{ - "otel-exporter-header-name": "X-Custom-Header", + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-name": "X-Custom-H$eader", "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller:nginx", }, }, - expectedLoadModule: false, - msg: "no endpoint, headers set", + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, header value invalid", }, { configMap: &v1.ConfigMap{ @@ -1528,12 +1570,34 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { "otel-trace-in-http": "invalid", }, }, - expectedLoadModule: true, - msg: "endpoint set, invalid trace flag", + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, trace flag invalid", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller:nginx", + "otel-trace-in-http": "true", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: true, + msg: "partially invalid, header name missing, trace in http set", }, } - isPlus := true + isPlus := false hasAppProtect := false hasAppProtectDos := false hasTLSPassthrough := false @@ -1549,6 +1613,21 @@ func TestOpenTelemetryConfigurationInvalid(t *testing.T) { if result.MainOtelLoadModule != test.expectedLoadModule { t.Errorf("MainOtelLoadModule: want %v, got %v", test.expectedLoadModule, result.MainOtelLoadModule) } + if result.MainOtelExporterEndpoint != test.expectedExporterEndpoint { + t.Errorf("MainOtelExporterEndpoint: want %q, got %q", test.expectedExporterEndpoint, result.MainOtelExporterEndpoint) + } + if result.MainOtelExporterHeaderName != test.expectedExporterHeaderName { + t.Errorf("MainOtelExporterHeaderName: want %q, got %q", test.expectedExporterHeaderName, result.MainOtelExporterHeaderName) + } + if result.MainOtelExporterHeaderValue != test.expectedExporterHeaderValue { + t.Errorf("MainOtelExporterHeaderValue: want %q, got %q", test.expectedExporterHeaderValue, result.MainOtelExporterHeaderValue) + } + if result.MainOtelServiceName != test.expectedServiceName { + t.Errorf("MainOtelServiceName: want %q, got %q", test.expectedServiceName, result.MainOtelServiceName) + } + if result.MainOtelTraceInHTTP != test.expectedTraceInHTTP { + t.Errorf("MainOtelTraceInHTTP: want %v, got %v", test.expectedTraceInHTTP, result.MainOtelTraceInHTTP) + } }) } } From a05419c773f7caae4a78c2cab2465477940adb94 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 20 May 2025 12:05:38 +0100 Subject: [PATCH 47/55] add additional comment to configmap --- examples/shared-examples/otel/nginx-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/shared-examples/otel/nginx-config.yaml b/examples/shared-examples/otel/nginx-config.yaml index 8b3cb5c65b..4113972b1f 100644 --- a/examples/shared-examples/otel/nginx-config.yaml +++ b/examples/shared-examples/otel/nginx-config.yaml @@ -8,4 +8,4 @@ data: otel-service-name: "nginx-ingress-controller:nginx" otel-exporter-header-name: "x-otel-header" otel-exporter-header-value: "otel-header-value" - # otel-trace-in-http: "true" + # otel-trace-in-http: "true" # Uncomment to enable tracing at the HTTP level From fc727e53178cb07329264a9a2ada1bf4c7f0263f Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 20 May 2025 12:25:26 +0100 Subject: [PATCH 48/55] update plus modules --- site/content/technical-specifications.md | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/site/content/technical-specifications.md b/site/content/technical-specifications.md index 2e151e08d5..efe16d4b60 100644 --- a/site/content/technical-specifications.md +++ b/site/content/technical-specifications.md @@ -65,22 +65,22 @@ _NGINX Plus images include NGINX Plus R34._ NGINX Plus images are available through the F5 Container registry `private-registry.nginx.com`, explained in the [Get the NGINX Ingress Controller image with JWT]({{}}) and [Get the F5 Registry NGINX Ingress Controller image]({{}}) topics. {{< bootstrap-table "table table-striped table-bordered table-responsive" >}} -|
Name
|
Base image
|
Third-party modules
| F5 Container Registry Image | Architectures | +|
Name
|
Base image
|
Additional modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.21`` | NGINX Plus & JavaScript | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.21`` | NGINX Plus & JavaScript

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | -|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF

NGINX Plus & JavaScript

FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5

NGINX Plus & JavaScript

FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Debian-based image | ``debian:12-slim`` | NGINX Plus & JavaScript | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` | NGINX App Protect WAF

NGINX Plus & JavaScript

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5

NGINX Plus & JavaScript | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS

NGINX Plus & JavaScript | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS

NGINX Plus & JavaScript | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Alpine-based image | ``alpine:3.21`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.21`` | NJS (NGINX JavaScript)
OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | +|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Debian-based image | ``debian:12-slim`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | |Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5 and NGINX Plus JavaScript module | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | {{% /bootstrap-table %}} --- From 33efb5e79557adce1dd36e28f37eae75a26027ba Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 20 May 2025 12:32:51 +0100 Subject: [PATCH 49/55] tryout unordered list for modules --- site/content/technical-specifications.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/content/technical-specifications.md b/site/content/technical-specifications.md index efe16d4b60..1223d2957c 100644 --- a/site/content/technical-specifications.md +++ b/site/content/technical-specifications.md @@ -67,8 +67,8 @@ NGINX Plus images are available through the F5 Container registry `private-regis {{< bootstrap-table "table table-striped table-bordered table-responsive" >}} |
Name
|
Base image
|
Additional modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.21`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.21`` | NJS (NGINX JavaScript)
OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | +|Alpine-based image | ``alpine:3.21`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.21`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | |Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | |Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | |Debian-based image | ``debian:12-slim`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | @@ -76,7 +76,7 @@ NGINX Plus images are available through the F5 Container registry `private-regis |Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | |Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | |Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | |Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | |Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | |Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | From 89c924d1502d7e6e344cdbace02485c9088ceba1 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 20 May 2025 12:41:50 +0100 Subject: [PATCH 50/55] complete list of modules in tech specs --- site/content/technical-specifications.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/site/content/technical-specifications.md b/site/content/technical-specifications.md index 1223d2957c..69d53160c0 100644 --- a/site/content/technical-specifications.md +++ b/site/content/technical-specifications.md @@ -69,18 +69,18 @@ NGINX Plus images are available through the F5 Container registry `private-regis | ---| ---| --- | --- | --- | |Alpine-based image | ``alpine:3.21`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | |Alpine-based image with FIPS inside | ``alpine:3.21`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | -|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry

FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Debian-based image | ``debian:12-slim`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` | NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NJS (NGINX JavaScript)

OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` |
  • NGINX App Protect WAF
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` |
  • NGINX App Protect WAF v5
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Debian-based image | ``debian:12-slim`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` |
  • NGINX App Protect WAF
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` |
  • NGINX App Protect WAF v5
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` |
  • NGINX App Protect DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` |
  • NGINX App Protect WAF and DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` |
  • NGINX App Protect WAF
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` |
  • NGINX App Protect WAF v5
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` |
  • NGINX App Protect DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` |
  • NGINX App Protect WAF and DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | {{% /bootstrap-table %}} --- From c8779362bc8e0480268bd60142d52f028da2eabf Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 20 May 2025 12:48:31 +0100 Subject: [PATCH 51/55] replace unordered list with single line break --- site/content/technical-specifications.md | 28 ++++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/site/content/technical-specifications.md b/site/content/technical-specifications.md index 69d53160c0..c9d7262d06 100644 --- a/site/content/technical-specifications.md +++ b/site/content/technical-specifications.md @@ -67,20 +67,20 @@ NGINX Plus images are available through the F5 Container registry `private-regis {{< bootstrap-table "table table-striped table-bordered table-responsive" >}} |
Name
|
Base image
|
Additional modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.21`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.21`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | -|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` |
  • NGINX App Protect WAF
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` |
  • NGINX App Protect WAF v5
  • NJS (NGINX JavaScript)
  • OpenTelemetry
  • FIPS module and OpenSSL configuration
| `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Debian-based image | ``debian:12-slim`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` |
  • NGINX App Protect WAF
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` |
  • NGINX App Protect WAF v5
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` |
  • NGINX App Protect DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` |
  • NGINX App Protect WAF and DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` |
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` |
  • NGINX App Protect WAF
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` |
  • NGINX App Protect WAF v5
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` |
  • NGINX App Protect DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` |
  • NGINX App Protect WAF and DoS
  • NJS (NGINX JavaScript)
  • OpenTelemetry
| `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Alpine-based image | ``alpine:3.21`` | NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.21`` | NJS (NGINX JavaScript)
OpenTelemetry
FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | +|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF
NJS (NGINX JavaScript)
OpenTelemetry
FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5
NJS (NGINX JavaScript)
OpenTelemetry
FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Debian-based image | ``debian:12-slim`` | NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` | NGINX App Protect WAF
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` | NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | {{% /bootstrap-table %}} --- From 93b48df64a6bb9ba72ecbdc10114d552b841c4f7 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Thu, 22 May 2025 11:45:52 +0100 Subject: [PATCH 52/55] fix rebase issue --- internal/configs/version1/nginx-plus.tmpl | 38 ----------------------- 1 file changed, 38 deletions(-) diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9355d503e6..b9ae81b126 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -162,44 +162,6 @@ http { {{- end}} {{- end}} - {{- if .MainOtelLoadModule }} - otel_exporter { - endpoint {{ .MainOtelExporterEndpoint }}; - {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} - header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; - {{- end }} - {{- if .MainOtelExporterTrustedCA }} - trusted_certificate /etc/nginx/secrets/{{ .MainOtelExporterTrustedCA }}; - {{- end }} - } - - {{ if .MainOtelServiceName}} - otel_service_name {{ .MainOtelServiceName }}; - {{- end }} - {{ if .MainOtelGlobalTraceEnabled }} - otel_trace on; - {{- end}} - {{- end}} - - {{- if .MainOtelLoadModule}} - otel_exporter { - endpoint {{ .MainOtelExporterEndpoint}}; - {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} - header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; - {{- end }} - {{- if .MainOtelExporterTrustedCA}} - # trusted_certificate ; - {{- end }} - } - - {{- if .MainOtelServiceName}} - otel_service_name {{ .MainOtelServiceName }}; - {{- end }} - {{- if .MainOtelGlobalTraceEnabled }} - otel_trace on; - {{- end}} - {{- end}} - {{ $resolverIPV6HTTPBool := boolToPointerBool .ResolverIPV6 -}} {{ makeResolver .ResolverAddresses .ResolverValid $resolverIPV6HTTPBool }} {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} From 73798d5895ff86ad0bbab60203ada8632795d1ed Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Thu, 22 May 2025 11:55:26 +0100 Subject: [PATCH 53/55] remove unused image import --- build/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/build/Dockerfile b/build/Dockerfile index 53d38a18c8..aa6457f672 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -11,7 +11,6 @@ ARG PACKAGE_REPO=pkgs.nginx.com ############################################# Base images containing libs for FIPS ############################################# -FROM ghcr.io/nginx/dependencies/nginx-ubi-ppc64le:nginx-1.27.4@sha256:fff4dde599b89cb22e5cea5d8cfba8c47bcedaa8e6fa549f5fe74a89c733aa2f AS ubi-ppc64le FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi8@sha256:7ddf7a566cb0c44e21b3652704e49c7f43e4d18dd875e1bbd4c4ec19d0b70989 AS ubi8-packages FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi9@sha256:a5dc7c5fef28271ecb651801785b18702e57b5ecd7cc8aa8867982c193d27d5b AS ubi9-packages FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.19@sha256:2a7f8451110b588b733e4cb8727a48153057b1debac5c78ef8a539ff63712fa1 AS alpine-fips-3.19 From 22359ed8cdc0d34ca49ad101b8ed09c5224714cb Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Tue, 27 May 2025 18:12:22 +0100 Subject: [PATCH 54/55] remove duplicate flag set --- internal/configs/configmaps.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index d35d66de8b..d0bf1bb3b5 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -823,7 +823,6 @@ func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams * cfgParams.MainOtelExporterHeaderName = "" cfgParams.MainOtelExporterHeaderValue = "" cfgParams.MainOtelServiceName = "" - cfgParams.MainOtelTraceInHTTP = false } if !otelValid { From 2586816c417b6c86c23a53da087be09d37224059 Mon Sep 17 00:00:00 2001 From: Paul Abel Date: Wed, 28 May 2025 10:15:05 +0100 Subject: [PATCH 55/55] address feedback --- .github/workflows/update-docker-sha.yml | 3 ++- .../global-configuration/configmap-resource.md | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update-docker-sha.yml b/.github/workflows/update-docker-sha.yml index f9723cf7fe..8858a5c6f4 100644 --- a/.github/workflows/update-docker-sha.yml +++ b/.github/workflows/update-docker-sha.yml @@ -62,7 +62,8 @@ jobs: ARGS="--exclude ${{ github.event.inputs.excludes }}" fi .github/scripts/docker-updater.sh ./build/Dockerfile $ARGS - .github/scripts/docker-updater.sh ./build/dependencies/Dockerfile.ubi $ARGS + .github/scripts/docker-updater.sh ./build/dependencies/Dockerfile.ubi8 $ARGS + .github/scripts/docker-updater.sh ./build/dependencies/Dockerfile.ubi9 $ARGS .github/scripts/docker-updater.sh ./tests/Dockerfile $ARGS files=$(git diff --name-only) if [[ $files == *"Dockerfile"* ]]; then diff --git a/site/content/configuration/global-configuration/configmap-resource.md b/site/content/configuration/global-configuration/configmap-resource.md index 715792d0eb..51cc96349e 100644 --- a/site/content/configuration/global-configuration/configmap-resource.md +++ b/site/content/configuration/global-configuration/configmap-resource.md @@ -230,11 +230,11 @@ If you encounter the error `error [emerg] 13#13: "zone_sync" directive is duplic {{}} |ConfigMap Key | Description | Default | Example | | ---| ---| ---| --- | -|*otel-exporter-endpoint* | OTLP/gRPC endpoint that will accept [OpenTelemetry](https://opentelemetry.io) data. Set `otel-trace-in-http` to *True* to enable OpenTelemetry at global level. Example: *"https://otel-collector:4317"*. Note: requires the Ingress Controller image with OpenTelemetry module. | | | -|*otel-exporter-header-name* | The name of a custom HTTP header to add to telemetry export request. Example: *"X-custom-header"*. `otel-exporter-endpoint` and `otel-exporter-header-value` required. | | | -|*otel-exporter-header-value* | The value of a custom HTTP header to add to telemetry export request.Example: *"custom-value"*. `otel-exporter-endpoint` and `otel-exporter-header-name` required. | | | -|*otel-service-name* | Sets the `service.name` attribute of the OTel resource. Example: *"nginx-ingress-controller:nginx"*. `otel-exporter-endpoint` required. | | | -| *otel-trace-in-http* | Enables [OpenTelemetry](https://opentelemetry.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Set this to *False* to enable OpenTelemetry for individual routes with snippets. `otel-exporter-endpoint` required. | *False* | | +|*otel-exporter-endpoint* | OTLP/gRPC endpoint that will accept [OpenTelemetry](https://opentelemetry.io) data. Set `otel-trace-in-http` to *"true"* to enable OpenTelemetry at the global level. | N/A | *"https://otel-collector:4317"* | +|*otel-exporter-header-name* | The name of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-value` required. | N/A | *"X-custom-header"* | +|*otel-exporter-header-value* | The value of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-name` required. | N/A | *"custom-value"* | +|*otel-service-name* | Sets the `service.name` attribute of the OTel resource. `otel-exporter-endpoint` required. | N/A | *"nginx-ingress-controller:nginx"* | +| *otel-trace-in-http* | Enables [OpenTelemetry](https://opentelemetry.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Set this to *"false"* to enable OpenTelemetry for individual routes with snippets. `otel-exporter-endpoint` required. | *"false"* | *"true"* | |*opentracing* | Removed in v5.0.0. Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs]({{< relref "/installation/integrations/opentracing.md" >}}) for more information. | *False* | | |*opentracing-tracer* | Removed in v5.0.0. Sets the path to the vendor tracer binary plugin. | N/A | | |*opentracing-tracer-config* | Removed in v5.0.0. Sets the tracer configuration in JSON format. | N/A | |