diff --git a/.github/data/matrix-smoke-nap.json b/.github/data/matrix-smoke-nap.json index cd2bcba29a..e0c8c1ac3b 100644 --- a/.github/data/matrix-smoke-nap.json +++ b/.github/data/matrix-smoke-nap.json @@ -5,7 +5,7 @@ "image": "ubi-8-plus-nap", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_policies_allow", + "marker": "'appprotect_waf_policies_allow or otel'", "platforms": "linux/amd64" }, { @@ -21,7 +21,7 @@ "image": "alpine-plus-nap-fips", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_policies_grpc", + "marker": "'appprotect_waf_policies_grpc or otel'", "platforms": "linux/amd64" }, { @@ -29,7 +29,7 @@ "image": "debian-plus-nap", "type": "plus", "nap_modules": "waf", - "marker": "'appprotect_watch or appprotect_batch or appprotect_integration or appprotect_waf_policies_vsr'", + "marker": "'appprotect_watch or appprotect_batch or appprotect_integration or appprotect_waf_policies_vsr or otel'", "platforms": "linux/amd64" }, { @@ -37,7 +37,7 @@ "image": "debian-plus-nap-v5", "type": "plus", "nap_modules": "waf", - "marker": "appprotect_waf_v5", + "marker": "'appprotect_waf_v5 or otel'", "platforms": "linux/amd64" }, { @@ -61,7 +61,7 @@ "image": "ubi-9-plus-nap", "type": "plus", "nap_modules": "dos", - "marker": "dos_learning", + "marker": "'dos_learning or otel'", "platforms": "linux/amd64" }, { diff --git a/.github/data/matrix-smoke-oss.json b/.github/data/matrix-smoke-oss.json index a15b9b8937..6c402dc04f 100644 --- a/.github/data/matrix-smoke-oss.json +++ b/.github/data/matrix-smoke-oss.json @@ -5,77 +5,77 @@ "image": "debian", "type": "oss", "marker": "'ingresses and not annotations and not basic_auth and not hsts and not watch_namespace and not wildcard_tls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "ingresses 2/2", "image": "debian", "type": "oss", "marker": "'annotations or basic_auth or hsts or watch_namespace or wildcard_tls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR 1/3", "image": "alpine", "type": "oss", "marker": "'vsr and not vsr_upstream and not vsr_grpc and not vsr_status and not vsr_canary and not vsr_routing and not vsr_api and not vsr_redirects and not vsr_rewrite and not vsr_canned and not vsr_basic'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR 2/3", "image": "alpine", "type": "oss", "marker": "'vsr_basic or vsr_canned or vsr_rewrite or vsr_redirects or vsr_upstream'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VSR 3/3", "image": "alpine", "type": "oss", "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "policies 1/2", "image": "alpine", "type": "oss", "marker": "'policies and not policies_rl and not policies_ac and not policies_jwt and not policies_mtls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "policies 2/2", "image": "alpine", "type": "oss", - "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "marker": "'policies_rl or policies_ac or policies_jwt or policies_mtls or otel'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 1/3", "image": "debian", "type": "oss", "marker": "'vs and not vs_ipv6 and not vs_rewrite and not vs_responses and not vs_grpc and not vs_redirects and not vs_externalname and not vs_externaldns and not vs_certmanager and not vs_api and not vs_backup and not vs_use_cluster_ip and not vs_canary and not vs_upstream and not vs_config_map'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 2/3", "image": "debian", "type": "oss", "marker": "'vs_grpc or vs_redirects or vs_externalname or vs_externaldns or vs_api or vs_backup or vs_use_cluster_ip or vs_canary or vs_upstream or vs_config_map'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "platforms": "linux/arm64, linux/amd64" }, { "label": "VS 3/3", "image": "debian", "type": "oss", - "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager'", - "platforms": "linux/arm, linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "marker": "'vs_responses or vs_ipv6 or vs_rewrite or vs_certmanager or otel'", + "platforms": "linux/arm64, linux/amd64" }, { "label": "TS", "image": "ubi", "type": "oss", - "marker": "ts", - "platforms": "linux/arm64, linux/amd64, linux/ppc64le, linux/s390x" + "marker": "'ts or otel'", + "platforms": "linux/arm64, linux/amd64" } ], "k8s": [] diff --git a/.github/data/matrix-smoke-plus.json b/.github/data/matrix-smoke-plus.json index 247ad5370a..4bded89736 100644 --- a/.github/data/matrix-smoke-plus.json +++ b/.github/data/matrix-smoke-plus.json @@ -25,7 +25,7 @@ "label": "TS", "image": "debian-plus", "type": "plus", - "marker": "ts", + "marker": "'ts or otel'", "platforms": "linux/arm64, linux/amd64" }, { @@ -60,7 +60,7 @@ "label": "VSR 3/3", "image": "alpine-plus", "type": "plus", - "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc'", + "marker": "'vsr_api or vsr_routing or vsr_canary or vsr_status or vsr_grpc or otel'", "platforms": "linux/arm64, linux/amd64" }, { @@ -74,7 +74,7 @@ "label": "policies 2/3", "image": "ubi-9-plus", "type": "plus", - "marker": "'policies_ac or policies_jwt or policies_mtls'", + "marker": "'policies_ac or policies_jwt or policies_mtls or otel'", "platforms": "linux/arm64, linux/amd64, linux/s390x" }, { diff --git a/.github/workflows/build-ubi-dependency.yml b/.github/workflows/build-ubi-dependency.yml index 9ac4b0e7e1..c48218be9e 100644 --- a/.github/workflows/build-ubi-dependency.yml +++ b/.github/workflows/build-ubi-dependency.yml @@ -1,92 +1,36 @@ -name: Build UBI ppc64le Dependency +name: Build UBI c-ares Dependency on: push: branches: - main paths: - - build/dependencies/Dockerfile.ubi + - build/dependencies/Dockerfile.ubi8 + - build/dependencies/Dockerfile.ubi9 + - .github/workflows/build-ubi-dependency.yml workflow_dispatch: - inputs: - nginx_version: - type: string - description: "NGINX Version to build for" - required: false - force: - type: boolean - description: "Force rebuild" - required: false - default: false env: - IMAGE_NAME: ghcr.io/nginx/dependencies/nginx-ubi-ppc64le + IMAGE_NAME: ghcr.io/nginx/dependencies/nginx-ubi concurrency: - group: ${{ github.ref_name }}-ubi-ppc64le-build + group: ${{ github.ref_name }}-ubi-build cancel-in-progress: true permissions: contents: read jobs: - checks: - name: Check versions - runs-on: ubuntu-22.04 - permissions: - packages: read - contents: read - strategy: - fail-fast: false - outputs: - nginx_version: ${{ steps.var.outputs.nginx_version }} - njs_version: ${{ steps.var.outputs.njs_version }} - target_exists: ${{ steps.var.outputs.target_image_exists }} - steps: - - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - - name: Login to GitHub Container Registry - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Output Variables - id: var - run: | - if [ -n "${{ inputs.nginx_version }}" ]; then - nginx_v=${{ inputs.nginx_version }} - else - nginx_v=$(grep -m1 'FROM nginx:' Outputs -------------------------------" - echo "NJS_VERSION=$njs" - echo "nginx_version=${nginx_v}" - echo "njs_version=${njs}" - echo "target_image_exists=${target_image_exists}" - echo "nginx_version=${nginx_v}" >> $GITHUB_OUTPUT - echo "njs_version=${njs}" >> $GITHUB_OUTPUT - echo "target_image_exists=${target_image_exists}" >> $GITHUB_OUTPUT - build-binaries: name: Build Binary Container Image - if: ${{ needs.checks.outputs.target_exists != 'true' || inputs.force }} - needs: checks runs-on: ubuntu-22.04 permissions: packages: write contents: read strategy: fail-fast: false + matrix: + tag: ["ubi8", "ubi9"] steps: - name: Checkout Repository uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -113,28 +57,23 @@ jobs: images: | name=${{ env.IMAGE_NAME }},enable=true tags: | - type=raw,value=nginx-${{ needs.checks.outputs.nginx_version }},enable=true + type=raw,value=${{ matrix.tag }},enable=true env: DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index - name: Build and push uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - file: ./build/dependencies/Dockerfile.ubi + file: ./build/dependencies/Dockerfile.${{ matrix.tag }} context: "." pull: true push: true - # build multi-arch so that it can be mounted from any image - # even though only ppc64le will contain binaries platforms: "linux/amd64,linux/arm64" tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} annotations: ${{ steps.meta.outputs.annotations }} - cache-from: type=gha,scope=nginx-ubi-ppc64le - cache-to: type=gha,scope=nginx-ubi-ppc64le,mode=max + cache-from: type=gha,scope=nginx-${{ matrix.tag }} + cache-to: type=gha,scope=nginx-${{ matrix.tag }},mode=max target: final sbom: false provenance: mode=max - build-args: | - NGINX=${{ needs.checks.outputs.nginx_version }} - NJS=${{ needs.checks.outputs.njs_version }} diff --git a/.github/workflows/update-docker-sha.yml b/.github/workflows/update-docker-sha.yml index f9723cf7fe..8858a5c6f4 100644 --- a/.github/workflows/update-docker-sha.yml +++ b/.github/workflows/update-docker-sha.yml @@ -62,7 +62,8 @@ jobs: ARGS="--exclude ${{ github.event.inputs.excludes }}" fi .github/scripts/docker-updater.sh ./build/Dockerfile $ARGS - .github/scripts/docker-updater.sh ./build/dependencies/Dockerfile.ubi $ARGS + .github/scripts/docker-updater.sh ./build/dependencies/Dockerfile.ubi8 $ARGS + .github/scripts/docker-updater.sh ./build/dependencies/Dockerfile.ubi9 $ARGS .github/scripts/docker-updater.sh ./tests/Dockerfile $ARGS files=$(git diff --name-only) if [[ $files == *"Dockerfile"* ]]; then diff --git a/build/Dockerfile b/build/Dockerfile index dd93724484..2fa4ae11a6 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -11,7 +11,8 @@ ARG PACKAGE_REPO=pkgs.nginx.com ############################################# Base images containing libs for FIPS ############################################# -FROM ghcr.io/nginx/dependencies/nginx-ubi-ppc64le:nginx-1.27.4@sha256:fff4dde599b89cb22e5cea5d8cfba8c47bcedaa8e6fa549f5fe74a89c733aa2f AS ubi-ppc64le +FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi8@sha256:7ddf7a566cb0c44e21b3652704e49c7f43e4d18dd875e1bbd4c4ec19d0b70989 AS ubi8-packages +FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi9@sha256:a5dc7c5fef28271ecb651801785b18702e57b5ecd7cc8aa8867982c193d27d5b AS ubi9-packages FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.19@sha256:2a7f8451110b588b733e4cb8727a48153057b1debac5c78ef8a539ff63712fa1 AS alpine-fips-3.19 FROM ghcr.io/nginx/alpine-fips:0.2.4-alpine3.21@sha256:5221dec2e33436f2586c743c7aa3ef4626c0ec54184dc3364d101036d4f4a060 AS alpine-fips-3.21 FROM redhat/ubi9-minimal:9.5@sha256:a50731d3397a4ee28583f1699842183d4d24fadcc565c4688487af9ee4e13a44 AS ubi-minimal @@ -21,15 +22,25 @@ FROM golang:1.24-alpine@sha256:ef18ee7117463ac1055f5a370ed18b8750f01589f13ea0b48 ############################################# Base image for Alpine ############################################# FROM nginx:1.27.4-alpine@sha256:4ff102c5d78d254a6f0da062b3cf39eaf07f01eec0927fd21e219d0af8bc0591 AS alpine -RUN apk add --no-cache libcap libstdc++ +RUN printf "%s%s%s\n" "http://nginx.org/packages/mainline/alpine/v" `egrep -o '^[0-9]+\.[0-9]+' /etc/alpine-release` "/main" >> /etc/apk/repositories \ + && apk add --no-cache libcap libstdc++ nginx-module-otel \ + && sed -i -e '/nginx.org/d' /etc/apk/repositories ############################################# Base image for Debian ############################################# FROM nginx:1.27.4@sha256:09369da6b10306312cd908661320086bf87fbae1b6b0c49a1f50ba531fef2eab AS debian RUN apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y libcap2-bin - + && apt-get install --no-install-recommends --no-install-suggests -y \ + libcap2-bin curl gnupg2 ca-certificates lsb-release debian-archive-keyring \ + && curl https://nginx.org/keys/nginx_signing.key | gpg --dearmor > /usr/share/keyrings/nginx-archive-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/nginx-archive-keyring.gpg] \ + http://nginx.org/packages/mainline/debian `lsb_release -cs` nginx" > /etc/apt/sources.list.d/nginx.list \ + && printf "%s" "Package: *\nPin: origin nginx.org\nPin: release o=nginx\nPin-Priority: 900\n" > /etc/apt/preferences.d/99nginx \ + && apt-get update \ + && apt-get install --no-install-recommends --no-install-suggests -y nginx-module-otel \ + && apt-get purge --auto-remove -y gnupg2 lsb-release curl \ + && rm -rf /var/lib/apt/lists/* /etc/apt/preferences.d/99nginx /etc/apt/sources.list.d/nginx.list ############################################# NGINX files ############################################# FROM scratch AS nginx-files @@ -109,7 +120,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/apk/cert.pem,mode=0644 \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ export $(cat /tmp/user_agent) \ && printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check libcap libcurl \ + && apk add --no-cache nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check libcap libcurl \ && mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ldconfig /usr/local/lib/ \ && sed -i -e '/nginx.com/d' /etc/apk/repositories @@ -151,7 +162,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://pkgs.nginx.com/app-protect-security-updates/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -187,7 +198,7 @@ RUN --mount=type=bind,from=alpine-fips-3.19,target=/tmp/fips/ \ printf "%s\n" "https://${PACKAGE_REPO}/plus/${NGINX_PLUS_VERSION}/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/app-protect-x-plus/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ && printf "%s\n" "https://${PACKAGE_REPO}/nginx-agent/alpine/v$(grep -E -o '^[0-9]+\.[0-9]+' /etc/alpine-release)/main" >> /etc/apk/repositories \ - && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apk add --no-cache libcap-utils libcurl nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then apk add --no-cache nginx-agent; fi \ && mkdir -p /usr/ssl \ && cp -av /tmp/fips/usr/lib/ossl-modules/fips.so /usr/lib/ossl-modules/fips.so \ @@ -226,7 +237,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && gpg --dearmor -o /usr/share/keyrings/app-protect-archive-keyring.gpg /tmp/app-protect-security-updates.key \ && cp /tmp/nginx-plus.sources /etc/apt/sources.list.d/nginx-plus.sources \ && apt-get update \ - && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && apt-get install --no-install-recommends --no-install-suggests -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && apt-get purge --auto-remove -y gpg \ && mkdir -p /etc/nginx/reporting/ \ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ @@ -332,23 +343,14 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_signing.key \ --mount=type=bind,from=nginx-files,src=ubi-setup.sh,target=/usr/local/bin/ubi-setup.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ - --mount=type=bind,from=ubi-ppc64le,src=/,target=/ubi-bin/ \ - ubi-setup.sh; \ - if [ $(uname -p) = ppc64le ] || [ $(uname -p) = s390x ]; then \ - rpm -qa --queryformat "%{NAME}\n" | sort > pkgs-installed \ - && microdnf --nodocs --setopt=install_weak_deps=0 install -y diffutils dnf \ - && rpm -qa --queryformat "%{NAME}\n" | sort > pkgs-new \ - && dnf install -y /ubi-bin/*.rpm \ - && dnf -q repoquery --resolve --requires --recursive --whatrequires nginx --queryformat "%{NAME}" > pkgs-nginx \ - && dnf --setopt=protected_packages= remove -y $(comm -13 pkgs-installed pkgs-new | comm -13 pkgs-nginx -) \ - && rm pkgs-installed pkgs-new pkgs-nginx; \ - else \ - printf "%s\n" "[nginx]" "name=nginx repo" \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ + ubi-setup.sh \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ + && printf "%s\n" "[nginx]" "name=nginx repo" \ "baseurl=https://nginx.org/packages/mainline/centos/9/\$basearch/" \ "gpgcheck=1" "enabled=1" "module_hotfixes=true" > /etc/yum.repos.d/nginx.repo \ - && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-image-filter nginx-module-xslt \ - && rm /etc/yum.repos.d/nginx.repo; \ - fi \ + && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel nginx-module-image-filter nginx-module-xslt \ + && rm /etc/yum.repos.d/nginx.repo \ && ubi-clean.sh @@ -366,9 +368,11 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=ubi-setup.sh,target=/usr/local/bin/ubi-setup.sh \ --mount=type=bind,from=nginx-files,src=ubi-clean.sh,target=/usr/local/bin/ubi-clean.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && ubi-setup.sh \ - && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ + && microdnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && ubi-clean.sh @@ -465,15 +469,17 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-waf.sh,target=/usr/local/bin/nap-waf.sh \ --mount=type=bind,from=nginx-files,src=agent.sh,target=/usr/local/bin/agent.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi8-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ cp /tmp/app-protect-8.repo /etc/yum.repos.d/app-protect-8.repo; \ fi \ && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ && subscription-manager register --org=${RHEL_ORGANIZATION} --activationkey=${RHEL_ACTIVATION_KEY} || true \ @@ -511,8 +517,10 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode --mount=type=bind,from=nginx-files,src=nap-waf.sh,target=/usr/local/bin/nap-waf.sh \ --mount=type=bind,from=nginx-files,src=agent.sh,target=/usr/local/bin/agent.sh \ --mount=type=bind,from=nginx-files,src=tracking.info,target=/tmp/nginx/reporting/tracking.info \ + --mount=type=bind,from=ubi8-packages,src=/,target=/ubi-bin/ \ mkdir -p /etc/nginx/reporting/ && cp -av /tmp/nginx/reporting/tracking.info /etc/nginx/reporting/tracking.info \ && source /tmp/rhel_license \ + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ && if [ -z "${NAP_MODULES##*waf*}" ]; then \ cp /tmp/app-protect-8.repo /etc/yum.repos.d/app-protect-8.repo; \ fi \ @@ -520,7 +528,7 @@ RUN --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode && groupadd --system --gid 101 nginx \ && useradd --system --gid nginx --no-create-home --home-dir /nonexistent --comment "nginx user" --shell /bin/false --uid 101 nginx \ && rpm --import /tmp/nginx_signing.key \ - && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-fips-check \ + && dnf --nodocs install -y nginx-plus nginx-plus-module-njs nginx-plus-module-otel nginx-plus-module-fips-check \ && if [ "${NGINX_AGENT}" = "true" ]; then dnf --nodocs install -y nginx-agent; fi \ ## end of duplicated code && sed -i 's/\(def in_container():\)/\1\n return False/g' /usr/lib64/python*/*-packages/rhsm/config.py \ diff --git a/build/dependencies/Dockerfile.ubi b/build/dependencies/Dockerfile.ubi deleted file mode 100644 index ecc5862f4d..0000000000 --- a/build/dependencies/Dockerfile.ubi +++ /dev/null @@ -1,33 +0,0 @@ -# syntax=docker/dockerfile:1.8 -FROM nginx:1.27.4@sha256:09369da6b10306312cd908661320086bf87fbae1b6b0c49a1f50ba531fef2eab AS nginx - -FROM redhat/ubi9:9.5@sha256:d07a5e080b8a9b3624d3c9cfbfada9a6baacd8e6d4065118f0e80c71ad518044 AS rpm-build -ARG NGINX -ARG NJS -ENV NGINX_VERSION=${NGINX} -ENV NJS_VERSION=${NJS} -RUN mkdir -p /nginx/; \ - # only build for ppc64le but make multiarch image for mounting - [ $(uname -p) = x86_64 ] && exit 0; \ - [ $(uname -p) = aarch64 ] && exit 0; \ - rpm --import https://nginx.org/keys/nginx_signing.key \ - && MINOR_VERSION=$(echo ${NGINX_VERSION} | cut -d '.' -f 2) \ - && if [ $(( $MINOR_VERSION % 2)) -eq 0 ]; then echo mainline=""; else mainline="mainline/"; fi \ - && printf "%s\n" "[nginx]" "name=nginx src repo" \ - "baseurl=https://nginx.org/packages/${mainline}centos/9/SRPMS" \ - "gpgcheck=1" "enabled=1" "module_hotfixes=true" >> /etc/yum.repos.d/nginx.repo \ - && dnf install rpm-build gcc make dnf-plugins-core which -y \ - && dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm \ - && nginxPackages=" \ - nginx-${NGINX_VERSION} \ - nginx-module-xslt-${NGINX_VERSION} \ - nginx-module-image-filter-${NGINX_VERSION} \ - nginx-module-njs-${NGINX_VERSION}+${NJS_VERSION} \ - " \ - && dnf download --source ${nginxPackages} \ - && dnf builddep -y --srpm nginx*.rpm \ - && rpmbuild --rebuild --nodebuginfo nginx*.rpm \ - && cp /root/rpmbuild/RPMS/$(arch)/* /nginx/ - -FROM scratch AS final -COPY --link --from=rpm-build /nginx / diff --git a/build/dependencies/Dockerfile.ubi8 b/build/dependencies/Dockerfile.ubi8 new file mode 100644 index 0000000000..3cb3dc7c01 --- /dev/null +++ b/build/dependencies/Dockerfile.ubi8 @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1.8 +FROM redhat/ubi8@sha256:244e9858f9d8a2792a3dceb850b4fa8fdbd67babebfde42587bfa919d5d1ecef AS rpm-build +RUN mkdir -p /rpms/ \ + && dnf install rpm-build gcc make cmake -y \ + && rpmbuild --rebuild --nodebuginfo https://mirror.stream.centos.org/9-stream/BaseOS/source/tree/Packages/c-ares-1.19.1-1.el9.src.rpm \ + && cp /root/rpmbuild/RPMS/$(arch)/* /rpms/ \ + && rm -rf /rpms/*devel* + +FROM scratch AS final +COPY --link --from=rpm-build /rpms / diff --git a/build/dependencies/Dockerfile.ubi9 b/build/dependencies/Dockerfile.ubi9 new file mode 100644 index 0000000000..25ef55a9fb --- /dev/null +++ b/build/dependencies/Dockerfile.ubi9 @@ -0,0 +1,10 @@ +# syntax=docker/dockerfile:1.8 +FROM redhat/ubi9:9.5@sha256:d07a5e080b8a9b3624d3c9cfbfada9a6baacd8e6d4065118f0e80c71ad518044 AS rpm-build +RUN mkdir -p /rpms/ \ + && dnf install rpm-build gcc make cmake -y \ + && rpmbuild --rebuild --nodebuginfo https://mirror.stream.centos.org/9-stream/BaseOS/source/tree/Packages/c-ares-1.19.1-1.el9.src.rpm \ + && cp /root/rpmbuild/RPMS/$(arch)/* /rpms/ \ + && rm -rf /rpms/*devel* + +FROM scratch AS final +COPY --link --from=rpm-build /rpms / diff --git a/examples/shared-examples/otel/README.md b/examples/shared-examples/otel/README.md new file mode 100644 index 0000000000..6ef1214658 --- /dev/null +++ b/examples/shared-examples/otel/README.md @@ -0,0 +1,82 @@ +# Learn how to use OpenTelemetry with F5 NGINX Ingress Controller + +NGINX Ingress Controller supports [OpenTelemetry](https://opentelemetry.io/) with the NGINX module [ngx_otel_module](https://nginx.org/en/docs/ngx_otel_module.html). + +## Prerequisites + +1. Use a NGINX Ingress Controller image that contains OpenTelemetry. + + - All NGINX Ingress Controller v5.1 images or later will contain support for `ngx_otel_module`. + - Alternatively, you follow [Build NGINX Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/installation/build-nginx-ingress-controller/) using `debian-image` (or `alpine-image` or `ubi-image`) for NGINX or `debian-image-plus` (or `alpine-image-plus`or `ubi-image-plus`) for NGINX Plus. + +1. Enable snippets annotations by setting the [`enable-snippets`](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-enable-snippets) command-line argument to true. + +1. Load the OpenTelemetry module. + + You need to load the module using the following ConfigMap key: + + - `otel-exporter-endpoint`: sets the endpoint to export your OpenTelemetry traces to. + + The following example shows how to use this to export data to an OpenTelemetry collector running in your cluster: + + ```yaml + otel-exporter-endpoint: "http://otel-collector.default.svc.cluster.local:4317" + ``` + +## Enable OpenTelemetry globally + +To enable OpenTelemetry globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `otel-trace-in-http` ConfigMap key to `true`: + +```yaml +otel-trace-in-http: "true" +``` + +## Enable or disable OpenTelemetry per Ingress resource + +You can use annotations to enable or disable OpenTelemetry for a specific Ingress resource. As mentioned in the prerequisites section, `otel-exporter-endpoint` must be configured. + +Consider the following two cases: + +### OpenTelemetry is globally disabled + +1. To enable OpenTelemetry for a specific Ingress resource, use the server snippet annotation: + + ```yaml + nginx.org/server-snippets: | + otel_trace on; + ``` + +1. To enable OpenTelemetry for specific paths: + + - You need to use [Mergeable Ingress resources](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration) + - You need to use the location snippets annotation to enable OpenTelemetry for the paths of a specific Minion Ingress resource: + + ```yaml + nginx.org/location-snippets: | + otel_trace on; + ``` + +### OpenTelemetry is globally enabled + +1. To disable OpenTelemetry for a specific Ingress resource, use the server snippet annotation: + + ```yaml + nginx.org/server-snippets: | + otel_trace off; + ``` + +1. To disable OpenTelemetry for specific paths: + + - You need to use [Mergeable Ingress resources](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration) + - You need to use the location snippets annotation to disable OpenTelemetry for the paths of a specific Minion Ingress resource: + + ```yaml + nginx.org/location-snippets: | + otel_trace off; + ``` + +## Customize OpenTelemetry + +You can customize OpenTelemetry through the supported [OpenTelemetry module directives](https://nginx.org/en/docs/ngx_otel_module.html). Use the `location-snippets` ConfigMap keys or annotations to insert those directives into the generated NGINX configuration. + +> Note: At present, the additional directives in the `otel_exporter` block cannot be modified with snippets. diff --git a/examples/shared-examples/otel/nginx-config.yaml b/examples/shared-examples/otel/nginx-config.yaml new file mode 100644 index 0000000000..4113972b1f --- /dev/null +++ b/examples/shared-examples/otel/nginx-config.yaml @@ -0,0 +1,11 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-service-name: "nginx-ingress-controller:nginx" + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" + # otel-trace-in-http: "true" # Uncomment to enable tracing at the HTTP level diff --git a/internal/configs/config_params.go b/internal/configs/config_params.go index ec8d0c8a62..c73310056d 100644 --- a/internal/configs/config_params.go +++ b/internal/configs/config_params.go @@ -34,6 +34,12 @@ type ConfigParams struct { MainLogFormat []string MainLogFormatEscaping string MainMainSnippets []string + MainOtelLoadModule bool + MainOtelTraceInHTTP bool + MainOtelExporterEndpoint string + MainOtelExporterHeaderName string + MainOtelExporterHeaderValue string + MainOtelServiceName string MainServerNamesHashBucketSize string MainServerNamesHashMaxSize string MainStreamLogFormat []string diff --git a/internal/configs/configmaps.go b/internal/configs/configmaps.go index 91a010c752..d0bf1bb3b5 100644 --- a/internal/configs/configmaps.go +++ b/internal/configs/configmaps.go @@ -9,6 +9,8 @@ import ( "strings" "time" + "github.com/nginx/kubernetes-ingress/internal/validation" + "github.com/nginx/kubernetes-ingress/internal/configs/commonhelpers" v1 "k8s.io/api/core/v1" @@ -16,7 +18,7 @@ import ( "github.com/nginx/kubernetes-ingress/internal/configs/version1" nl "github.com/nginx/kubernetes-ingress/internal/logger" - "github.com/nginx/kubernetes-ingress/internal/validation" + k8s_validation "k8s.io/apimachinery/pkg/util/validation" ) const ( @@ -530,6 +532,11 @@ func ParseConfigMap(ctx context.Context, cfgm *v1.ConfigMap, nginxPlus bool, has } } + _, otelErr := parseConfigMapOpenTelemetry(l, cfgm, cfgParams, eventLog) + if otelErr != nil { + configOk = false + } + if hasAppProtect { if appProtectFailureModeAction, exists := cfgm.Data["app-protect-failure-mode-action"]; exists { if appProtectFailureModeAction == "pass" || appProtectFailureModeAction == "drop" { @@ -740,6 +747,91 @@ func parseConfigMapZoneSync(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams *Confi return &cfgParams.ZoneSync, nil } +//nolint:gocyclo +func parseConfigMapOpenTelemetry(l *slog.Logger, cfgm *v1.ConfigMap, cfgParams *ConfigParams, eventLog record.EventRecorder) (*ConfigParams, error) { + otelValid := true + + if otelExporterEndpoint, exists := cfgm.Data["otel-exporter-endpoint"]; exists { + otelExporterEndpoint = strings.TrimSpace(otelExporterEndpoint) + if otelExporterEndpoint != "" { + cfgParams.MainOtelExporterEndpoint = otelExporterEndpoint + } + } + + if otelExporterHeaderName, exists := cfgm.Data["otel-exporter-header-name"]; exists { + otelExporterHeaderName = strings.TrimSpace(otelExporterHeaderName) + if otelExporterHeaderName != "" { + errorMessages := k8s_validation.IsHTTPHeaderName(otelExporterHeaderName) + if len(errorMessages) > 0 { + errorText := fmt.Sprintf("ConfigMap %s/%s: invalid value for 'otel-exporter-header-name': %q, %v", cfgm.GetNamespace(), cfgm.GetName(), otelExporterHeaderName, errorMessages) + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + otelValid = false + } else { + cfgParams.MainOtelExporterHeaderName = otelExporterHeaderName + } + } + } + + if otelExporterHeaderValue, exists := cfgm.Data["otel-exporter-header-value"]; exists { + otelExporterHeaderValue = strings.TrimSpace(otelExporterHeaderValue) + if otelExporterHeaderValue != "" { + cfgParams.MainOtelExporterHeaderValue = otelExporterHeaderValue + } + } + + if otelServiceName, exists := cfgm.Data["otel-service-name"]; exists { + otelServiceName = strings.TrimSpace(otelServiceName) + if otelServiceName != "" { + cfgParams.MainOtelServiceName = otelServiceName + } + } + + if otelTraceInHTTP, exists, err := GetMapKeyAsBool(cfgm.Data, "otel-trace-in-http", cfgm); exists { + if err != nil { + nl.Error(l, err) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, err.Error()) + otelValid = false + } + cfgParams.MainOtelTraceInHTTP = otelTraceInHTTP + } + + if (cfgParams.MainOtelExporterHeaderName != "" && cfgParams.MainOtelExporterHeaderValue == "") || + (cfgParams.MainOtelExporterHeaderName == "" && cfgParams.MainOtelExporterHeaderValue != "") { + cfgParams.MainOtelExporterHeaderName = "" + cfgParams.MainOtelExporterHeaderValue = "" + errorText := "Both 'otel-exporter-header-name' and 'otel-exporter-header-value' must be set or neither" + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + otelValid = false + } + + if cfgParams.MainOtelExporterEndpoint != "" { + cfgParams.MainOtelLoadModule = true + } + + if cfgParams.MainOtelExporterEndpoint == "" && + (cfgParams.MainOtelExporterHeaderName != "" || + cfgParams.MainOtelExporterHeaderValue != "" || + cfgParams.MainOtelServiceName != "" || + cfgParams.MainOtelTraceInHTTP) { + errorText := "ConfigMap key 'otel-exporter-endpoint' is required when other otel fields are set" + nl.Error(l, errorText) + eventLog.Event(cfgm, v1.EventTypeWarning, nl.EventReasonInvalidValue, errorText) + otelValid = false + cfgParams.MainOtelTraceInHTTP = false + cfgParams.MainOtelExporterHeaderName = "" + cfgParams.MainOtelExporterHeaderValue = "" + cfgParams.MainOtelServiceName = "" + } + + if !otelValid { + return nil, errors.New("invalid OpenTelemetry configuration") + } + + return cfgParams, nil +} + // ParseMGMTConfigMap parses the mgmt block ConfigMap into MGMTConfigParams. // //nolint:gocyclo @@ -913,6 +1005,12 @@ func GenerateNginxMainConfig(staticCfgParams *StaticConfigParams, config *Config NginxStatus: staticCfgParams.NginxStatus, NginxStatusAllowCIDRs: staticCfgParams.NginxStatusAllowCIDRs, NginxStatusPort: staticCfgParams.NginxStatusPort, + MainOtelLoadModule: config.MainOtelLoadModule, + MainOtelGlobalTraceEnabled: config.MainOtelTraceInHTTP, + MainOtelExporterEndpoint: config.MainOtelExporterEndpoint, + MainOtelExporterHeaderName: config.MainOtelExporterHeaderName, + MainOtelExporterHeaderValue: config.MainOtelExporterHeaderValue, + MainOtelServiceName: config.MainOtelServiceName, ProxyProtocol: config.ProxyProtocol, ResolverAddresses: config.ResolverAddresses, ResolverIPV6: config.ResolverIPV6, diff --git a/internal/configs/configmaps_test.go b/internal/configs/configmaps_test.go index 517c384780..397e6da9ca 100644 --- a/internal/configs/configmaps_test.go +++ b/internal/configs/configmaps_test.go @@ -1377,6 +1377,261 @@ func TestParseZoneSyncResolverIPV6MapResolverIPV6(t *testing.T) { } } +func TestOpenTelemetryConfigurationSuccess(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + expectedLoadModule bool + expectedExporterEndpoint string + expectedExporterHeaderName string + expectedExporterHeaderValue string + expectedServiceName string + expectedTraceInHTTP bool + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-service-name": "nginx-ingress-controller:nginx", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + msg: "endpoint set, minimal config", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-trusted-ca": "otel-ca-secret", + "otel-exporter-header-name": "X-Custom-Header", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller:nginx", + "otel-trace-in-http": "true", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "X-Custom-Header", + expectedExporterHeaderValue: "custom-value", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: true, + msg: "endpoint set, full config", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{}, + }, + expectedLoadModule: false, + expectedExporterEndpoint: "", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "", + expectedTraceInHTTP: false, + msg: "no config", + }, + } + + isPlus := true + hasAppProtect := false + hasAppProtectDos := false + hasTLSPassthrough := false + expectedConfigOk := true + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + result, configOk := ParseConfigMap(context.Background(), test.configMap, isPlus, + hasAppProtect, hasAppProtectDos, hasTLSPassthrough, makeEventLogger()) + if configOk != expectedConfigOk { + t.Errorf("configOk: want %v, got %v", expectedConfigOk, configOk) + } + if result.MainOtelLoadModule != test.expectedLoadModule { + t.Errorf("MainOtelLoadModule: want %v, got %v", test.expectedLoadModule, result.MainOtelLoadModule) + } + if result.MainOtelExporterEndpoint != test.expectedExporterEndpoint { + t.Errorf("MainOtelExporterEndpoint: want %q, got %q", test.expectedExporterEndpoint, result.MainOtelExporterEndpoint) + } + if result.MainOtelExporterHeaderName != test.expectedExporterHeaderName { + t.Errorf("MainOtelExporterHeaderName: want %q, got %q", test.expectedExporterHeaderName, result.MainOtelExporterHeaderName) + } + if result.MainOtelExporterHeaderValue != test.expectedExporterHeaderValue { + t.Errorf("MainOtelExporterHeaderValue: want %q, got %q", test.expectedExporterHeaderValue, result.MainOtelExporterHeaderValue) + } + if result.MainOtelServiceName != test.expectedServiceName { + t.Errorf("MainOtelServiceName: want %q, got %q", test.expectedServiceName, result.MainOtelServiceName) + } + if result.MainOtelTraceInHTTP != test.expectedTraceInHTTP { + t.Errorf("MainOtelTraceInHTTP: want %v, got %v", test.expectedTraceInHTTP, result.MainOtelTraceInHTTP) + } + }) + } +} + +func TestOpenTelemetryConfigurationInvalid(t *testing.T) { + t.Parallel() + tests := []struct { + configMap *v1.ConfigMap + expectedLoadModule bool + expectedExporterEndpoint string + expectedExporterHeaderName string + expectedExporterHeaderValue string + expectedServiceName string + expectedTraceInHTTP bool + msg string + }{ + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "", + "otel-service-name": "nginx-ingress-controller:nginx", + }, + }, + expectedLoadModule: false, + expectedExporterEndpoint: "", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "", + expectedTraceInHTTP: false, + msg: "invalid, endpoint missing, service name set", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-header-name": "X-Custom-Header", + "otel-exporter-header-value": "custom-value", + }, + }, + expectedLoadModule: false, + expectedExporterEndpoint: "", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "", + expectedTraceInHTTP: false, + msg: "invalid, endpoint missing, header name and value set", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-name": "X-Custom-Header", + "otel-service-name": "nginx-ingress-controller:nginx", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, header value missing", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller:nginx", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, header name missing", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-name": "X-Custom-H$eader", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller:nginx", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, header value invalid", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-service-name": "nginx-ingress-controller:nginx", + "otel-trace-in-http": "invalid", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: false, + msg: "partially invalid, trace flag invalid", + }, + { + configMap: &v1.ConfigMap{ + Data: map[string]string{ + "otel-exporter-endpoint": "https://otel-collector:4317", + "otel-exporter-header-value": "custom-value", + "otel-service-name": "nginx-ingress-controller:nginx", + "otel-trace-in-http": "true", + }, + }, + expectedLoadModule: true, + expectedExporterEndpoint: "https://otel-collector:4317", + expectedExporterHeaderName: "", + expectedExporterHeaderValue: "", + expectedServiceName: "nginx-ingress-controller:nginx", + expectedTraceInHTTP: true, + msg: "partially invalid, header name missing, trace in http set", + }, + } + + isPlus := false + hasAppProtect := false + hasAppProtectDos := false + hasTLSPassthrough := false + expectedConfigOk := false + + for _, test := range tests { + t.Run(test.msg, func(t *testing.T) { + result, configOk := ParseConfigMap(context.Background(), test.configMap, isPlus, + hasAppProtect, hasAppProtectDos, hasTLSPassthrough, makeEventLogger()) + if configOk != expectedConfigOk { + t.Errorf("configOk: want %v, got %v", expectedConfigOk, configOk) + } + if result.MainOtelLoadModule != test.expectedLoadModule { + t.Errorf("MainOtelLoadModule: want %v, got %v", test.expectedLoadModule, result.MainOtelLoadModule) + } + if result.MainOtelExporterEndpoint != test.expectedExporterEndpoint { + t.Errorf("MainOtelExporterEndpoint: want %q, got %q", test.expectedExporterEndpoint, result.MainOtelExporterEndpoint) + } + if result.MainOtelExporterHeaderName != test.expectedExporterHeaderName { + t.Errorf("MainOtelExporterHeaderName: want %q, got %q", test.expectedExporterHeaderName, result.MainOtelExporterHeaderName) + } + if result.MainOtelExporterHeaderValue != test.expectedExporterHeaderValue { + t.Errorf("MainOtelExporterHeaderValue: want %q, got %q", test.expectedExporterHeaderValue, result.MainOtelExporterHeaderValue) + } + if result.MainOtelServiceName != test.expectedServiceName { + t.Errorf("MainOtelServiceName: want %q, got %q", test.expectedServiceName, result.MainOtelServiceName) + } + if result.MainOtelTraceInHTTP != test.expectedTraceInHTTP { + t.Errorf("MainOtelTraceInHTTP: want %v, got %v", test.expectedTraceInHTTP, result.MainOtelTraceInHTTP) + } + }) + } +} + func makeEventLogger() record.EventRecorder { return record.NewFakeRecorder(1024) } diff --git a/internal/configs/version1/__snapshots__/template_test.snap b/internal/configs/version1/__snapshots__/template_test.snap index 4a2a657e12..66a7b6f4b7 100644 --- a/internal/configs/version1/__snapshots__/template_test.snap +++ b/internal/configs/version1/__snapshots__/template_test.snap @@ -4263,6 +4263,154 @@ stream { --- +[TestExecuteTemplate_ForMainForNGINXWithOtel - 1] +worker_processes ; + +daemon off; + +error_log stderr ; +pid /var/lib/nginx/nginx.pid; +load_module modules/ngx_otel_module.so; +load_module modules/ngx_fips_check_module.so; + +load_module modules/ngx_http_js_module.so; + +events { + worker_connections ; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + map_hash_max_size ; + map_hash_bucket_size ; + + js_import /etc/nginx/njs/apikey_auth.js; + js_set $apikey_auth_hash apikey_auth.hash; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + map $upstream_trailer_grpc_status $grpc_status { + default $upstream_trailer_grpc_status; + '' $sent_http_grpc_status; + } + + access_log ; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout ; + keepalive_requests 0; + + #gzip on; + + server_names_hash_max_size ; + + + variables_hash_bucket_size 0; + variables_hash_max_size 0; + + map $request_uri $request_uri_no_args { + "~^(?P[^?]*)(\?.*)?$" $path; + } + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + map $http_upgrade $vs_connection_header { + default upgrade; + '' $default_connection_header; + } + otel_exporter { + endpoint https://otel-collector:4317; + header X-Custom-Header "custom-value"; + } + + + otel_service_name nginx-ingress-controller:nginx; + + otel_trace on; + + + + + server { + # required to support the Websocket protocol in VirtualServer/VirtualServerRoutes + set $default_connection_header ""; + set $resource_type ""; + set $resource_name ""; + set $resource_namespace ""; + set $service ""; + + listen 0 default_server;listen [::]:0 default_server; + listen 0 ssl default_server; + listen [::]:0 ssl default_server; + ssl_certificate /etc/nginx/secrets/default; + ssl_certificate_key /etc/nginx/secrets/default; + + server_name _; + server_tokens ""; + + location / { + return ; + } + } + + # NGINX Plus API over unix socket + server { + listen unix:/var/lib/nginx/nginx-plus-api.sock; + access_log off; + + # $config_version_mismatch is defined in /etc/nginx/config-version.conf + location /configVersionCheck { + if ($config_version_mismatch) { + return 503; + } + return 200; + } + + location /api { + api write=on; + } + } + + include /etc/nginx/config-version.conf; + include /etc/nginx/conf.d/*.conf; + + server { + listen unix:/var/lib/nginx/nginx-418-server.sock; + access_log off; + + return 418; + } +} + +stream { + log_format stream-main '$remote_addr [$time_local] ' + '$protocol $status $bytes_sent $bytes_received ' + '$session_time "$ssl_preread_server_name"'; + + access_log /dev/stdout stream-main; + + + + map_hash_max_size ; + + include /etc/nginx/stream-conf.d/*.conf; +} + +mgmt { + license_token /license.jwt; + enforce_initial_report off; + deployment_context /etc/nginx/reporting/tracking.info; +} + +--- + [TestExecuteTemplate_ForMainForNGINXWithZoneSyncEnabledCustomPort - 1] worker_processes ; diff --git a/internal/configs/version1/config.go b/internal/configs/version1/config.go index 8b510e315f..106470b865 100644 --- a/internal/configs/version1/config.go +++ b/internal/configs/version1/config.go @@ -240,6 +240,12 @@ type MainConfig struct { NginxStatus bool NginxStatusAllowCIDRs []string NginxStatusPort int + MainOtelLoadModule bool + MainOtelGlobalTraceEnabled bool + MainOtelExporterEndpoint string + MainOtelExporterHeaderName string + MainOtelExporterHeaderValue string + MainOtelServiceName string ProxyProtocol bool ResolverAddresses []string ResolverIPV6 bool diff --git a/internal/configs/version1/nginx-plus.tmpl b/internal/configs/version1/nginx-plus.tmpl index 9b5a0738c5..b9ae81b126 100644 --- a/internal/configs/version1/nginx-plus.tmpl +++ b/internal/configs/version1/nginx-plus.tmpl @@ -12,6 +12,9 @@ daemon off; error_log stderr {{.ErrorLogLevel}}; pid /var/lib/nginx/nginx.pid; +{{- if .MainOtelLoadModule}} +load_module modules/ngx_otel_module.so; +{{- end}} {{- if .AppProtectLoadModule}} load_module modules/ngx_http_app_protect_module.so; {{- end}} @@ -143,6 +146,22 @@ http { ssl_dhparam {{.SSLDHParam}}; {{- end}} + {{- if .MainOtelLoadModule }} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint }}; + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} + } + + {{ if .MainOtelServiceName}} + otel_service_name {{ .MainOtelServiceName }}; + {{- end }} + {{ if .MainOtelGlobalTraceEnabled }} + otel_trace on; + {{- end}} + {{- end}} + {{ $resolverIPV6HTTPBool := boolToPointerBool .ResolverIPV6 -}} {{ makeResolver .ResolverAddresses .ResolverValid $resolverIPV6HTTPBool }} {{if .ResolverTimeout}}resolver_timeout {{.ResolverTimeout}};{{end}} diff --git a/internal/configs/version1/nginx.tmpl b/internal/configs/version1/nginx.tmpl index 5b290b381d..a5f8d7760b 100644 --- a/internal/configs/version1/nginx.tmpl +++ b/internal/configs/version1/nginx.tmpl @@ -11,6 +11,10 @@ daemon off; error_log stderr {{.ErrorLogLevel}}; pid /var/lib/nginx/nginx.pid; +{{- if .MainOtelLoadModule}} +load_module modules/ngx_otel_module.so; +{{- end}} + {{- if .MainSnippets}} {{range $value := .MainSnippets}} {{$value}}{{end}} @@ -104,6 +108,22 @@ http { ssl_dhparam {{.SSLDHParam}}; {{- end}} + {{- if .MainOtelLoadModule }} + otel_exporter { + endpoint {{ .MainOtelExporterEndpoint }}; + {{- if and .MainOtelExporterHeaderName .MainOtelExporterHeaderValue }} + header {{ .MainOtelExporterHeaderName }} "{{ .MainOtelExporterHeaderValue }}"; + {{- end }} + } + + {{- if .MainOtelServiceName}} + otel_service_name {{ .MainOtelServiceName }}; + {{- end }} + {{- if .MainOtelGlobalTraceEnabled }} + otel_trace on; + {{- end}} + {{- end}} + server { # required to support the Websocket protocol in VirtualServer/VirtualServerRoutes set $default_connection_header ""; diff --git a/internal/configs/version1/template_test.go b/internal/configs/version1/template_test.go index e21da7ec80..05430f1945 100644 --- a/internal/configs/version1/template_test.go +++ b/internal/configs/version1/template_test.go @@ -1080,6 +1080,36 @@ func TestExecuteTemplate_ForMainForNGINXWithZoneSyncEnabledCustomResolverAddress snaps.MatchSnapshot(t, buf.String()) } +func TestExecuteTemplate_ForMainForNGINXWithOtel(t *testing.T) { + t.Parallel() + + tmpl := newNGINXPlusMainTmpl(t) + buf := &bytes.Buffer{} + + err := tmpl.Execute(buf, mainCfgWithOTel) + t.Log(buf.String()) + + if err != nil { + t.Fatalf("Failed to write template %v", err) + } + + wantDirectives := []string{ + "otel_exporter {", + "endpoint https://otel-collector:4317;", + "header X-Custom-Header \"custom-value\";", + "otel_service_name nginx-ingress-controller:nginx;", + "otel_trace on;", + } + + mainConf := buf.String() + for _, want := range wantDirectives { + if !strings.Contains(mainConf, want) { + t.Errorf("want %q in generated config", want) + } + } + snaps.MatchSnapshot(t, buf.String()) +} + func TestExecuteTemplate_ForIngressForNGINXWithProxySetHeadersAnnotationWithDefaultValue(t *testing.T) { t.Parallel() @@ -2589,6 +2619,15 @@ var ( }, } + mainCfgWithOTel = MainConfig{ + MainOtelLoadModule: true, + MainOtelGlobalTraceEnabled: true, + MainOtelExporterEndpoint: "https://otel-collector:4317", + MainOtelExporterHeaderName: "X-Custom-Header", + MainOtelExporterHeaderValue: "custom-value", + MainOtelServiceName: "nginx-ingress-controller:nginx", + } + // Vars for Mergable Ingress Master - Minion tests coffeeUpstreamNginxPlus = Upstream{ diff --git a/pyproject.toml b/pyproject.toml index a2bdec0dcb..cfd6f488bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ markers = [ "ingresses", "multi_ns", "oidc", + "otel", "policies", "policies_rl", "policies_jwt", diff --git a/site/content/configuration/global-configuration/configmap-resource.md b/site/content/configuration/global-configuration/configmap-resource.md index 724f6bb05e..51cc96349e 100644 --- a/site/content/configuration/global-configuration/configmap-resource.md +++ b/site/content/configuration/global-configuration/configmap-resource.md @@ -230,6 +230,11 @@ If you encounter the error `error [emerg] 13#13: "zone_sync" directive is duplic {{}} |ConfigMap Key | Description | Default | Example | | ---| ---| ---| --- | +|*otel-exporter-endpoint* | OTLP/gRPC endpoint that will accept [OpenTelemetry](https://opentelemetry.io) data. Set `otel-trace-in-http` to *"true"* to enable OpenTelemetry at the global level. | N/A | *"https://otel-collector:4317"* | +|*otel-exporter-header-name* | The name of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-value` required. | N/A | *"X-custom-header"* | +|*otel-exporter-header-value* | The value of a custom HTTP header to add to telemetry export request. `otel-exporter-endpoint` and `otel-exporter-header-name` required. | N/A | *"custom-value"* | +|*otel-service-name* | Sets the `service.name` attribute of the OTel resource. `otel-exporter-endpoint` required. | N/A | *"nginx-ingress-controller:nginx"* | +| *otel-trace-in-http* | Enables [OpenTelemetry](https://opentelemetry.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Set this to *"false"* to enable OpenTelemetry for individual routes with snippets. `otel-exporter-endpoint` required. | *"false"* | *"true"* | |*opentracing* | Removed in v5.0.0. Enables [OpenTracing](https://opentracing.io) globally (for all Ingress, VirtualServer and VirtualServerRoute resources). Note: requires the Ingress Controller image with OpenTracing module and a tracer. See the [docs]({{< relref "/installation/integrations/opentracing.md" >}}) for more information. | *False* | | |*opentracing-tracer* | Removed in v5.0.0. Sets the path to the vendor tracer binary plugin. | N/A | | |*opentracing-tracer-config* | Removed in v5.0.0. Sets the tracer configuration in JSON format. | N/A | | diff --git a/site/content/technical-specifications.md b/site/content/technical-specifications.md index 2e151e08d5..c9d7262d06 100644 --- a/site/content/technical-specifications.md +++ b/site/content/technical-specifications.md @@ -65,22 +65,22 @@ _NGINX Plus images include NGINX Plus R34._ NGINX Plus images are available through the F5 Container registry `private-registry.nginx.com`, explained in the [Get the NGINX Ingress Controller image with JWT]({{}}) and [Get the F5 Registry NGINX Ingress Controller image]({{}}) topics. {{< bootstrap-table "table table-striped table-bordered table-responsive" >}} -|
Name
|
Base image
|
Third-party modules
| F5 Container Registry Image | Architectures | +|
Name
|
Base image
|
Additional modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.21`` | NGINX Plus & JavaScript | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.21`` | NGINX Plus & JavaScript

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | -|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF

NGINX Plus & JavaScript

FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5

NGINX Plus & JavaScript

FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | -|Debian-based image | ``debian:12-slim`` | NGINX Plus & JavaScript | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` | NGINX App Protect WAF

NGINX Plus & JavaScript

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5

NGINX Plus & JavaScript | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS

NGINX Plus & JavaScript | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS

NGINX Plus & JavaScript | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5 and NGINX Plus JavaScript module | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Alpine-based image | ``alpine:3.21`` | NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.21`` | NJS (NGINX JavaScript)
OpenTelemetry
FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | arm64
amd64 | +|Alpine-based image with NGINX App Protect WAF & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF
NJS (NGINX JavaScript)
OpenTelemetry
FIPS module and OpenSSL configuration | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Alpine-based image with NGINX App Protect WAF v5 & FIPS inside | ``alpine:3.19`` | NGINX App Protect WAF v5
NJS (NGINX JavaScript)
OpenTelemetry
FIPS module and OpenSSL configuration | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-alpine-fips` | amd64 | +|Debian-based image | ``debian:12-slim`` | NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:12-slim`` | NGINX App Protect WAF
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF v5 | ``debian:12-slim`` | NGINX App Protect WAF v5
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:12-slim`` | NGINX App Protect DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:12-slim`` | NGINX App Protect WAF and DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}` | amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` | NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic/nginx-plus-ingress:{{< nic-version >}}-ubi` | arm64
amd64 | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi9`` | NGINX App Protect WAF
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF v5 | ``redhat/ubi9`` | NGINX App Protect WAF v5
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-v5/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS
NJS (NGINX JavaScript)
OpenTelemetry | `nginx-ic-nap-dos/nginx-plus-ingress:{{< nic-version >}}-ubi` | amd64 | {{% /bootstrap-table %}} --- diff --git a/tests/Makefile b/tests/Makefile index 3b955a6f04..8fe5f96035 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,7 +1,7 @@ SHELL = /bin/bash ROOT_DIR = $(shell git rev-parse --show-toplevel) CONTEXT = -PULL_POLICY = IfNotPresent +PULL_POLICY ?= IfNotPresent DEPLOYMENT_TYPE = deployment SERVICE = nodeport NODE_IP = diff --git a/tests/data/otel/configmap-with-all-except-endpoint.yaml b/tests/data/otel/configmap-with-all-except-endpoint.yaml new file mode 100644 index 0000000000..9940afb925 --- /dev/null +++ b/tests/data/otel/configmap-with-all-except-endpoint.yaml @@ -0,0 +1,10 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" + otel-service-name: "nginx-ingress-controller:nginx" + otel-trace-in-http: "true" diff --git a/tests/data/otel/configmap-with-all.yaml b/tests/data/otel/configmap-with-all.yaml new file mode 100644 index 0000000000..3f0dbecbac --- /dev/null +++ b/tests/data/otel/configmap-with-all.yaml @@ -0,0 +1,11 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" + otel-service-name: "nginx-ingress-controller:nginx" + otel-trace-in-http: "true" diff --git a/tests/data/otel/configmap-with-endpoint.yaml b/tests/data/otel/configmap-with-endpoint.yaml new file mode 100644 index 0000000000..fc462e2c79 --- /dev/null +++ b/tests/data/otel/configmap-with-endpoint.yaml @@ -0,0 +1,7 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" diff --git a/tests/data/otel/configmap-with-header.yaml b/tests/data/otel/configmap-with-header.yaml new file mode 100644 index 0000000000..868b7017f5 --- /dev/null +++ b/tests/data/otel/configmap-with-header.yaml @@ -0,0 +1,9 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-name: "x-otel-header" + otel-exporter-header-value: "otel-header-value" diff --git a/tests/data/otel/configmap-with-only-header-name.yaml b/tests/data/otel/configmap-with-only-header-name.yaml new file mode 100644 index 0000000000..edac4d37cc --- /dev/null +++ b/tests/data/otel/configmap-with-only-header-name.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-name: "x-otel-header" diff --git a/tests/data/otel/configmap-with-only-header-value.yaml b/tests/data/otel/configmap-with-only-header-value.yaml new file mode 100644 index 0000000000..d7f95030cb --- /dev/null +++ b/tests/data/otel/configmap-with-only-header-value.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-exporter-header-value: "otel-header-value" diff --git a/tests/data/otel/configmap-with-otel-trace.yaml b/tests/data/otel/configmap-with-otel-trace.yaml new file mode 100644 index 0000000000..7cb1e2a87f --- /dev/null +++ b/tests/data/otel/configmap-with-otel-trace.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-trace-in-http: "true" diff --git a/tests/data/otel/configmap-with-service-name.yaml b/tests/data/otel/configmap-with-service-name.yaml new file mode 100644 index 0000000000..29e29439cd --- /dev/null +++ b/tests/data/otel/configmap-with-service-name.yaml @@ -0,0 +1,8 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: + otel-exporter-endpoint: "otel.example.com:4317" + otel-service-name: "nginx-ingress-controller:nginx" diff --git a/tests/data/otel/default-configmap.yaml b/tests/data/otel/default-configmap.yaml new file mode 100644 index 0000000000..a6a6c812b5 --- /dev/null +++ b/tests/data/otel/default-configmap.yaml @@ -0,0 +1,6 @@ +kind: ConfigMap +apiVersion: v1 +metadata: + name: nginx-config + namespace: nginx-ingress +data: diff --git a/tests/suite/test_otel.py b/tests/suite/test_otel.py new file mode 100644 index 0000000000..153527ca3a --- /dev/null +++ b/tests/suite/test_otel.py @@ -0,0 +1,473 @@ +import pytest +from settings import TEST_DATA +from suite.utils.resources_utils import ( + extract_block, + get_nginx_template_conf, + replace_configmap_from_yaml, + wait_before_test, +) + +WAIT_TIME = 1 +cm_default = f"{TEST_DATA}/otel/default-configmap.yaml" +cm_endpoint = f"{TEST_DATA}/otel/configmap-with-endpoint.yaml" +cm_header = f"{TEST_DATA}/otel/configmap-with-header.yaml" +cm_header_only_name = f"{TEST_DATA}/otel/configmap-with-only-header-name.yaml" +cm_header_only_value = f"{TEST_DATA}/otel/configmap-with-only-header-value.yaml" +cm_service_name = f"{TEST_DATA}/otel/configmap-with-service-name.yaml" +cm_otel_trace = f"{TEST_DATA}/otel/configmap-with-otel-trace.yaml" +cm_all = f"{TEST_DATA}/otel/configmap-with-all.yaml" +cm_all_except_endpoint = f"{TEST_DATA}/otel/configmap-with-all-except-endpoint.yaml" +otel_module = "modules/ngx_otel_module.so" +exporter = "otel.example.com:4317" +otel_exporter_header_name = "x-otel-header" +otel_exporter_header_value = "otel-header-value" +otel_service_name = "nginx-ingress-controller:nginx" +configmap_name = "nginx-config" + + +@pytest.mark.otel +class TestOtel: + + def test_otel_not_enabled( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts without otel configured in the `nginx-config` + 2. Ensure that the otel is not enabled in the nginx.conf + """ + + print("Step 1: apply default nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + + # Verify otel not present in nginx.conf + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + assert "otel" not in (nginx_config) + + def test_otel_endpoint( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `endpoint` is enabled in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_endpoint, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the endpoint is correctly configured") + assert f"endpoint {exporter};" in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + + def test_otel_header( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `header` is enabled in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_header, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the header is correctly configured") + assert f'header {otel_exporter_header_name} "{otel_exporter_header_value}";' in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_header_name_only( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `header` is not in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_header_only_name, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the header is not configured") + assert f"header" not in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_header_value_only( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `header` is not in the `otel_exporter` block. + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_header_only_value, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the header is not configured") + assert f"header" not in (exporter_block) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_service_name( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `service-name` is enabled in the nginx.conf + 5. Ensure that `otel_trace` is not configured + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_service_name, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the service-name is correctly configured") + assert f"otel_service_name {otel_service_name}" in (nginx_config) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_trace( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that `otel_trace` is configured in the nginx.conf + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_otel_trace, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that otel_trace is configured") + assert "otel_trace on;" in (nginx_config) + + print("Step 5: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_all( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with otel endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is loaded in the nginx.conf + 3. Ensure that the `otel_exporter` is enabled in the nginx.conf + 4. Ensure that the `endpoint` is enabled in the `otel_exporter` block. + 5. Ensure that the `header` is enabled in the `otel_exporter` block. + 6. Ensure that the `service-name` is enabled in the nginx.conf + 7. Ensure that `otel_trace` is configured in the nginx.conf + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_all, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is loaded") + assert otel_module in (nginx_config) + + exporter_block = extract_block(nginx_config, "otel_exporter") + + print("Step 3: Ensure that the otel_exporter is enabled") + assert "otel_exporter" in (exporter_block) + + print("Step 4: Ensure that the endpoint is correctly configured") + assert f"endpoint {exporter};" in (exporter_block) + + print("Step 5: Ensure that the header is correctly configured") + assert f'header {otel_exporter_header_name} "{otel_exporter_header_value}";' in (exporter_block) + + print("Step 6: Ensure that the service-name is correctly configured") + assert f"otel_service_name {otel_service_name}" in (nginx_config) + + print("Step 7: Ensure that otel_trace is configured") + assert "otel_trace on;" in (nginx_config) + + print("Step 8: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) + + def test_otel_all_except_endpoint( + self, + kube_apis, + ingress_controller_prerequisites, + ingress_controller, + ): + """ + Test: + 1. NIC starts with all otel configuration except endpoint configured in the `nginx-config` + 2. Ensure that the `ngx_otel_module.so` is not in the nginx.conf + 3. Ensure that the `otel_exporter` is not in the nginx.conf + 4. Ensure that the `service-name` is not in the nginx.conf + 5. Ensure that `otel_trace` is not in the nginx.conf + """ + configmap_name = "nginx-config" + + print("Step 1: apply nginx-config map") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_all_except_endpoint, + ) + + wait_before_test(WAIT_TIME) + nginx_config = get_nginx_template_conf( + kube_apis.v1, ingress_controller_prerequisites.namespace, print_log=False + ) + + print("Step 2: Ensure that the otel module is not loaded") + assert otel_module not in (nginx_config) + + print("Step 3: Ensure that the otel_exporter is not enabled") + assert "otel_exporter" not in (nginx_config) + + print("Step 4: Ensure that the service-name is not correctly configured") + assert f"otel_service_name {otel_service_name}" not in (nginx_config) + + print("Step 5: Ensure that otel_trace is not configured") + assert "otel_trace on;" not in (nginx_config) + + print("Step 6: reset the configmap to default") + replace_configmap_from_yaml( + kube_apis.v1, + configmap_name, + ingress_controller_prerequisites.namespace, + cm_default, + ) + wait_before_test(WAIT_TIME) diff --git a/tests/suite/utils/resources_utils.py b/tests/suite/utils/resources_utils.py index 1916bf4992..9a25cc0aea 100644 --- a/tests/suite/utils/resources_utils.py +++ b/tests/suite/utils/resources_utils.py @@ -1011,7 +1011,7 @@ def clear_file_contents(v1: CoreV1Api, file_path, pod_name, pod_namespace): ) -def get_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ic_pod_name=None) -> str: +def get_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ic_pod_name=None, print_log=True) -> str: """ Get contents of /etc/nginx/nginx.conf in the pod :param v1: CoreV1Api @@ -1022,7 +1022,7 @@ def get_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ic_pod_name=None) if ic_pod_name is None: ic_pod_name = get_first_pod_name(v1, ingress_namespace) file_path = "/etc/nginx/nginx.conf" - return get_file_contents(v1, file_path, ic_pod_name, ingress_namespace) + return get_file_contents(v1, file_path, ic_pod_name, ingress_namespace, print_log) def get_ingress_nginx_template_conf(v1: CoreV1Api, ingress_namespace, ingress_name, pod_name, pod_namespace) -> str: @@ -1070,6 +1070,19 @@ def get_ts_nginx_template_conf(v1: CoreV1Api, resource_namespace, resource_name, return get_file_contents(v1, file_path, pod_name, pod_namespace) +def extract_block(nginx_config, block_name): + """ + Extract a block of configuration from the nginx config file. + + :param nginx_config: The nginx config file content as a string. + :param block_name: The name of the block to extract. + :return: The extracted block as a string. + """ + start = nginx_config.find(block_name) + end = nginx_config.find("}", start) + 1 + return nginx_config[start:end] + + def create_example_app(kube_apis, app_type, namespace) -> None: """ Create a backend application.