diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1c0f5f6868..a8ed66a334 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,6 +6,10 @@ on: platforms: required: true type: string + build-os: + required: false + type: string + default: '' image: required: true type: string @@ -116,16 +120,17 @@ jobs: name=ghcr.io/${{ github.repository_owner }}/nginx-gateway-fabric/nginx,enable=${{ inputs.image == 'nginx' && github.event_name != 'pull_request' }} name=docker-mgmt.nginx.com/nginx-gateway-fabric/nginx-plus,enable=${{ inputs.image == 'plus' && github.event_name != 'pull_request' }} name=us-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/nginx-gateway-fabric/nginx-plus,enable=${{ inputs.image == 'plus' && github.event_name != 'pull_request' }} + name=ghcr.io/${{ github.repository_owner }}/nginx-gateway-fabric/operator,enable=${{ inputs.image == 'operator' && github.event_name != 'pull_request' }} name=localhost:5000/nginx-gateway-fabric/${{ inputs.image }} flavor: | latest=${{ (inputs.tag != '' && 'true') || 'auto' }} tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') && inputs.tag == '' }} - type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }} + type=semver,pattern={{version}},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=edge,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=schedule,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=pr,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=branch,suffix=-rc${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }},enable=${{ startsWith(github.ref, 'refs/heads/release') && inputs.tag == '' }} + type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} labels: | org.opencontainers.image.documentation=https://docs.nginx.com/nginx-gateway-fabric org.opencontainers.image.vendor=NGINX Inc @@ -143,7 +148,7 @@ jobs: - name: Build Docker Image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || '' }} + file: ${{ inputs.image == 'operator' && 'operators/Dockerfile' || (inputs.build-os != '' && format('build/{0}/Dockerfile{1}', inputs.build-os, inputs.image == 'nginx' && '.nginx' || inputs.image == 'plus' && '.nginxplus' || '') || format('build/Dockerfile{0}', inputs.image == 'nginx' && '.nginx' || inputs.image == 'plus' && '.nginxplus' || '')) }} context: "." target: ${{ inputs.image == 'ngf' && 'goreleaser' || '' }} tags: ${{ steps.meta.outputs.tags }} @@ -151,8 +156,8 @@ jobs: annotations: ${{ steps.meta.outputs.annotations }} push: ${{ !inputs.dry_run }} platforms: ${{ inputs.platforms }} - cache-from: type=gha,scope=${{ inputs.image }} - cache-to: type=gha,scope=${{ inputs.image }},mode=max + cache-from: type=gha,scope=${{ inputs.image }}${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + cache-to: type=gha,scope=${{ inputs.image }}${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }},mode=max pull: true no-cache: ${{ github.event_name != 'pull_request' }} sbom: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8db8183099..10b86dd949 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,10 @@ on: required: false type: string default: '' + operator_version: + required: false + type: string + default: '' dry_run: required: false type: boolean @@ -350,10 +354,12 @@ jobs: matrix: image: [ngf, nginx] platforms: ["linux/arm64, linux/amd64"] + build-os: ["", ubi] uses: ./.github/workflows/build.yml with: image: ${{ matrix.image }} platforms: ${{ matrix.platforms }} + build-os: ${{ matrix.build-os }} tag: ${{ inputs.release_version || '' }} dry_run: ${{ inputs.dry_run || false}} runner: ${{ github.repository_owner == 'nginx' && (inputs.is_production_release || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && 'ubuntu-24.04-amd64' || 'ubuntu-24.04' }} @@ -368,9 +374,14 @@ jobs: name: Build Plus images needs: [vars, binary] uses: ./.github/workflows/build.yml + strategy: + fail-fast: false + matrix: + build-os: ["", ubi] with: image: plus platforms: "linux/arm64, linux/amd64" + build-os: ${{ matrix.build-os }} tag: ${{ inputs.release_version || '' }} dry_run: ${{ inputs.dry_run || false }} runner: ${{ github.repository_owner == 'nginx' && (inputs.is_production_release || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && 'ubuntu-24.04-amd64' || 'ubuntu-24.04' }} @@ -381,6 +392,23 @@ jobs: id-token: write # for docker/login to login to NGINX registry secrets: inherit + build-operator: + name: Build Operator images + needs: [vars, binary] + uses: ./.github/workflows/build.yml + with: + image: operator + platforms: "linux/arm64, linux/amd64" + tag: ${{ inputs.operator_version || '' }} + dry_run: ${{ inputs.dry_run || false }} + runner: ${{ github.repository_owner == 'nginx' && (inputs.is_production_release || (github.event_name == 'push' && github.ref == 'refs/heads/main')) && 'ubuntu-24.04-amd64' || 'ubuntu-24.04' }} + permissions: + contents: read # for docker/build-push-action to read repo content + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + packages: write # for docker/build-push-action to push to GHCR + id-token: write # for docker/login to login to NGINX registry + secrets: inherit + functional-tests: name: Functional tests needs: [vars, build-oss, build-plus] @@ -388,6 +416,7 @@ jobs: fail-fast: false matrix: image: [nginx, plus] + build-os: ["", ubi] k8s-version: [ "${{ needs.vars.outputs.min_k8s_version }}", @@ -397,6 +426,7 @@ jobs: with: image: ${{ matrix.image }} k8s-version: ${{ matrix.k8s-version }} + build-os: ${{ matrix.build-os }} secrets: inherit permissions: contents: read @@ -408,6 +438,7 @@ jobs: fail-fast: false matrix: image: [nginx, plus] + build-os: ["", ubi] k8s-version: [ "${{ needs.vars.outputs.min_k8s_version }}", @@ -419,6 +450,7 @@ jobs: image: ${{ matrix.image }} k8s-version: ${{ matrix.k8s-version }} enable-experimental: ${{ matrix.enable-experimental }} + build-os: ${{ matrix.build-os }} production-release: ${{ inputs.is_production_release == true && (inputs.dry_run == false || inputs.dry_run == null) }} release_version: ${{ inputs.release_version }} secrets: inherit diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index 1a9480987d..fb6d7b909e 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -6,6 +6,10 @@ on: image: required: true type: string + build-os: + required: false + type: string + default: '' k8s-version: required: true type: string @@ -75,12 +79,12 @@ jobs: images: | name=ghcr.io/nginx/nginx-gateway-fabric tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') && !inputs.production-release }} - type=raw,value={{inputs.release_version}},enable=${{ inputs.production-release && inputs.release_version != '' }} + type=semver,pattern={{version}},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=edge,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=schedule,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=pr,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=branch,suffix=-rc${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }},enable=${{ startsWith(github.ref, 'refs/heads/release') && !inputs.production-release }} + type=raw,value={{ inputs.release_version }},enable=${{ inputs.production-release && inputs.release_version != '' }},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} - name: NGINX Docker meta id: nginx-meta @@ -89,12 +93,12 @@ jobs: images: | name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') && !inputs.production-release }} - type=raw,value={{inputs.release_version}},enable=${{ inputs.production-release && inputs.release_version != '' }} + type=semver,pattern={{version}},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=edge,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=schedule,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=pr,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=branch,suffix=-rc${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }},enable=${{ startsWith(github.ref, 'refs/heads/release') && !inputs.production-release }} + type=raw,value={{ inputs.release_version }},enable=${{ inputs.production-release && inputs.release_version != '' }},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} - name: Build binary uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 @@ -119,11 +123,11 @@ jobs: - name: Build NGINX Docker Image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + file: build${{ inputs.build-os != '' && format('/{0}', inputs.build-os) || '' }}/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || '' }} tags: ${{ steps.nginx-meta.outputs.tags }} context: "." load: true - cache-from: type=gha,scope=${{ inputs.image }} + cache-from: type=gha,scope=${{ inputs.image }}${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} pull: true build-args: | NJS_DIR=internal/controller/nginx/modules/src @@ -178,7 +182,7 @@ jobs: if: ${{ inputs.enable-experimental }} uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: conformance-profile-${{ inputs.image }}-${{ inputs.k8s-version }} + name: conformance-profile-${{ inputs.image }}-${{ inputs.k8s-version }}-${{ steps.ngf-meta.outputs.version }} path: ./tests/conformance-profile.yaml - name: Upload profile to release diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml index 2db23b9434..d092f3145c 100644 --- a/.github/workflows/functional.yml +++ b/.github/workflows/functional.yml @@ -9,6 +9,10 @@ on: k8s-version: required: true type: string + build-os: + required: false + type: string + default: '' defaults: run: @@ -61,11 +65,11 @@ jobs: images: | name=ghcr.io/nginx/nginx-gateway-fabric tags: | - type=semver,pattern={{version}} - type=schedule - type=edge - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + type=semver,pattern={{version}},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=schedule,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=edge,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=pr,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=branch,suffix=-rc${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }},enable=${{ startsWith(github.ref, 'refs/heads/release') }} - name: NGINX Docker meta id: nginx-meta @@ -74,11 +78,11 @@ jobs: images: | name=ghcr.io/nginx/nginx-gateway-fabric/${{ inputs.image == 'plus' && 'nginx-plus' || inputs.image }} tags: | - type=semver,pattern={{version}} - type=edge - type=schedule - type=ref,event=pr - type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }} + type=semver,pattern={{version}},suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=schedule,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=edge,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=pr,suffix=${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} + type=ref,event=branch,suffix=-rc${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }},enable=${{ startsWith(github.ref, 'refs/heads/release') }} - name: Build binary uses: goreleaser/goreleaser-action@e435ccd777264be153ace6237001ef4d979d3a7a # v6.4.0 @@ -103,11 +107,11 @@ jobs: - name: Build NGINX Docker Image uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: - file: build/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || ''}} + file: build${{ inputs.build-os != '' && format('/{0}', inputs.build-os) || '' }}/Dockerfile${{ inputs.image == 'nginx' && '.nginx' || '' }}${{ inputs.image == 'plus' && '.nginxplus' || '' }} tags: ${{ steps.nginx-meta.outputs.tags }} context: "." load: true - cache-from: type=gha,scope=${{ inputs.image }} + cache-from: type=gha,scope=${{ inputs.image }}${{ inputs.build-os != '' && format('-{0}', inputs.build-os) || '' }} pull: true build-args: | NJS_DIR=internal/controller/nginx/modules/src diff --git a/.github/workflows/production-release.yml b/.github/workflows/production-release.yml index 76471c9ec4..37617f7ee7 100644 --- a/.github/workflows/production-release.yml +++ b/.github/workflows/production-release.yml @@ -7,6 +7,11 @@ on: description: 'Release version (e.g., v2.0.3)' required: true type: string + operator-version: + description: 'Operator release version (e.g., v1.0.0). Optional' + required: false + type: string + default: '' dry_run: description: 'If true, does a dry run of the production workflow' required: false @@ -33,6 +38,7 @@ jobs: echo "Validating release from: ${GITHUB_REF}" INPUT_VERSION="${{ github.event.inputs.version }}" + INPUT_OPERATOR_VERSION="${{ github.event.inputs.operator-version }}" # Validate version format if [[ ! "${INPUT_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then @@ -41,8 +47,17 @@ jobs: exit 1 fi + # Validate version format if operator version is provided + if [[ -n "${INPUT_OPERATOR_VERSION}" && ! "${INPUT_OPERATOR_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "❌ Invalid operator version format: ${INPUT_OPERATOR_VERSION}" + echo "Expected format: v1.2.3" + exit 1 + fi + + echo "✅ Valid release branch: ${GITHUB_REF}" echo "✅ Valid version format: ${INPUT_VERSION}" + [[ -n "${INPUT_OPERATOR_VERSION}" ]] && echo "✅ Valid operator version format: ${INPUT_OPERATOR_VERSION}" - name: Checkout Repository uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 @@ -62,7 +77,7 @@ jobs: git tag -a "${VERSION}" -m "Release ${VERSION}" if [[ "${{ inputs.dry_run }}" == "true" ]]; then - echo "DRY RUN: Would push tag ${VERSION}" + echo "DRY RUN: Would push tag ${VERSION} and operator tag ${{ github.event.inputs.operator-version || '' }}" git push --dry-run origin "${VERSION}" else git push origin "${VERSION}" @@ -76,6 +91,7 @@ jobs: with: is_production_release: true release_version: ${{ github.event.inputs.version }} + operator_version: ${{ github.event.inputs.operator-version }} dry_run: ${{ github.event.inputs.dry_run }} secrets: inherit permissions: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 75f756d0df..63ba54bd29 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,6 @@ repos: - id: check-yaml args: [--allow-multiple-documents] exclude: (^charts/nginx-gateway-fabric/templates) - - id: check-added-large-files - id: check-merge-conflict - id: check-case-conflict - id: check-vcs-permalinks diff --git a/.yamllint.yaml b/.yamllint.yaml index e52cae4940..2ba44f7656 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -27,6 +27,8 @@ rules: spaces: consistent indent-sequences: consistent check-multi-line-strings: true + ignore: | + operators/**/* key-duplicates: enable key-ordering: disable line-length: @@ -38,6 +40,7 @@ rules: tests/suite/manifests/longevity/cronjob.yaml .goreleaser.yml charts/nginx-gateway-fabric/ + operators/config/crd/bases/gateway.nginx.org_nginxgatewayfabrics.yaml new-line-at-end-of-file: enable new-lines: enable octal-values: disable diff --git a/Makefile b/Makefile index e8a5902449..1391c54abd 100644 --- a/Makefile +++ b/Makefile @@ -43,6 +43,7 @@ HELM_SCHEMA_VERSION = 0.18.1 PREFIX ?= nginx-gateway-fabric## The name of the NGF image. For example, nginx-gateway-fabric NGINX_PREFIX ?= $(PREFIX)/nginx## The name of the nginx image. For example: nginx-gateway-fabric/nginx NGINX_PLUS_PREFIX ?= $(PREFIX)/nginx-plus## The name of the nginx plus image. For example: nginx-gateway-fabric/nginx-plus +BUILD_OS ?= ## The OS of the nginx image. Possible values: ubi and empty string, which defaults to alpine. NGINX_SERVICE_TYPE ?= NodePort## The type of the nginx service. Possible values: NodePort, LoadBalancer, ClusterIP PULL_POLICY ?= Never## The pull policy of the images. Possible values: Always, IfNotPresent, Never TAG ?= $(VERSION:v%=%)## The tag of the image. For example, 1.1.0 @@ -85,21 +86,21 @@ build-prod-ngf-image: build-ngf-image ## Build the NGF docker image for producti .PHONY: build-ngf-image build-ngf-image: check-for-docker build ## Build the NGF docker image - docker build --platform linux/$(GOARCH) --build-arg BUILD_AGENT=$(BUILD_AGENT) --target $(strip $(TARGET)) -f $(SELF_DIR)build/Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) + docker build --platform linux/$(GOARCH) --build-arg BUILD_AGENT=$(BUILD_AGENT) --target $(strip $(TARGET)) -f $(SELF_DIR)build/$(if $(BUILD_OS),$(BUILD_OS)/)Dockerfile -t $(strip $(PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) .PHONY: build-prod-nginx-image build-prod-nginx-image: build-nginx-image ## Build the custom nginx image for production .PHONY: build-nginx-image build-nginx-image: check-for-docker ## Build the custom nginx image - docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f $(SELF_DIR)build/Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) + docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) -f $(SELF_DIR)build/$(if $(BUILD_OS),$(BUILD_OS)/)Dockerfile.nginx -t $(strip $(NGINX_PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) .PHONY: build-prod-nginx-plus-image build-prod-nginx-plus-image: build-nginx-plus-image ## Build the custom nginx plus image for production .PHONY: build-nginx-plus-image build-nginx-plus-image: check-for-docker ## Build the custom nginx plus image - docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) $(strip $(NGINX_DOCKER_BUILD_PLUS_ARGS)) -f $(SELF_DIR)build/Dockerfile.nginxplus -t $(strip $(NGINX_PLUS_PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) + docker build --platform linux/$(GOARCH) $(strip $(NGINX_DOCKER_BUILD_OPTIONS)) $(strip $(NGINX_DOCKER_BUILD_PLUS_ARGS)) -f $(SELF_DIR)build/$(if $(BUILD_OS),$(BUILD_OS)/)Dockerfile.nginxplus -t $(strip $(NGINX_PLUS_PREFIX)):$(strip $(TAG)) $(strip $(SELF_DIR)) .PHONY: check-for-docker check-for-docker: ## Check if Docker is installed diff --git a/build/entrypoint.sh b/build/entrypoint.sh index 9e9552b338..3c05aebcd9 100755 --- a/build/entrypoint.sh +++ b/build/entrypoint.sh @@ -40,12 +40,12 @@ fi nginx_pid=$! SECONDS=0 - -while ! ps -ef | grep "nginx: master process" | grep -v grep; do - if ((SECONDS > 5)); then +while [[ ! -f /var/run/nginx.pid ]] && [[ ! -f /var/run/nginx/nginx.pid ]]; do + if ((SECONDS > 30)); then echo "couldn't find nginx master process" exit 1 fi + sleep 1 done # start nginx-agent, pass args diff --git a/build/ubi/Dockerfile b/build/ubi/Dockerfile new file mode 100644 index 0000000000..81cc76e417 --- /dev/null +++ b/build/ubi/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1.18 +FROM golang:1.25 AS builder + +WORKDIR /go/src/github.com/nginx/nginx-gateway-fabric + +COPY go.mod go.sum /go/src/github.com/nginx/nginx-gateway-fabric/ +RUN go mod download + +COPY . /go/src/github.com/nginx/nginx-gateway-fabric +RUN make build + +FROM golang:1.25 AS ca-certs-provider + +FROM redhat/ubi9-minimal:9.6 AS ngf-ubi-minimal +# CA certs are needed for telemetry report so that NGF can verify the server's certificate. +COPY --from=ca-certs-provider --link /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +USER 101:1001 +ARG BUILD_AGENT +ENV BUILD_AGENT=${BUILD_AGENT} +ENTRYPOINT [ "/usr/bin/gateway" ] + +FROM ngf-ubi-minimal AS container +COPY --from=builder /go/src/github.com/nginxinc/nginx-gateway-fabric/build/out/gateway /usr/bin/gateway + +FROM ngf-ubi-minimal AS local +COPY ./build/out/gateway /usr/bin/gateway + +FROM ngf-ubi-minimal AS goreleaser +ARG TARGETARCH +COPY dist/gateway_linux_$TARGETARCH*/gateway /usr/bin/gateway diff --git a/build/ubi/Dockerfile.nginx b/build/ubi/Dockerfile.nginx new file mode 100644 index 0000000000..5967bd5734 --- /dev/null +++ b/build/ubi/Dockerfile.nginx @@ -0,0 +1,69 @@ +# syntax=docker/dockerfile:1.18 +FROM scratch AS nginx-files + +# Repository and key files for UBI-based builds +ADD --link --chown=101:1001 https://nginx.org/keys/nginx_signing.key nginx_signing.key +ADD --link --chown=101:1001 build/ubi/repos/nginx.repo nginx.repo +ADD --link --chown=101:1001 build/ubi/repos/agent.repo agent.repo + +FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi9@sha256:01a32246761b9bbe47a6a29bcd8ca6e9b6e331b3bdfa372d8987b622276f7025 AS ubi9-packages + +FROM redhat/ubi9-minimal:9.6 AS ubi-nginx + +# renovate: datasource=github-tags depName=nginx/agent +ARG NGINX_AGENT_VERSION=v3.3.1 +ARG NJS_DIR +ARG NGINX_CONF_DIR +ARG BUILD_AGENT + +LABEL name="F5 NGINX Gateway Fabric NGINX OSS" \ + maintainer="kubernetes@nginx.com" \ + vendor="F5 NGINX" \ + summary="NGINX Gateway Fabric" \ + description="NGINX Gateway Fabric provides an implementation for the Gateway API using NGINX as the data plane." \ + org.nginx.ngf.image.build.agent="${BUILD_AGENT}" \ + io.k8s.description="NGINX Gateway Fabric provides an implementation for the Gateway API using NGINX as the data plane." \ + io.openshift.tags="nginx,gateway,kubernetes,openshift" + +COPY --link --chown=101:1001 LICENSE /licenses/ + +# Install NGINX with packages +RUN --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_signing.key \ + --mount=type=bind,from=nginx-files,src=nginx.repo,target=/etc/yum.repos.d/nginx.repo \ + --mount=type=bind,from=nginx-files,src=agent.repo,target=/etc/yum.repos.d/agent.repo \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ + # Import NGINX signing key + rpm --import /tmp/nginx_signing.key \ + # Install c-ares from the dependencies image (contains required libs) + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ + # Create nginx user with consistent UID/GID + && groupadd -g 1001 nginx \ + && useradd -r -u 101 -g nginx -s /sbin/nologin -d /var/cache/nginx nginx \ + # Install NGINX and modules including OTEL + && microdnf --nodocs install -y nginx nginx-module-njs nginx-module-otel \ + # Install nginx-agent + && microdnf --nodocs install -y nginx-agent-${NGINX_AGENT_VERSION#v}* \ + # Clean up (only remove what we can) + && microdnf clean all \ + && rm -rf /var/cache/yum + +# Configure directories and logging +RUN mkdir -p /var/run/nginx /usr/lib64/nginx/modules \ + # Forward request and error logs to docker log collector + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log + +# Set proper permissions for nginx user +RUN chown -R 101:1001 /etc/nginx /var/cache/nginx + +# Copy configuration files and scripts +COPY build/entrypoint.sh /agent/entrypoint.sh +COPY ${NJS_DIR}/ /usr/lib64/nginx/modules/njs/ +COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf +COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf +COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf + +# Switch to non-root user +USER 101:1001 + +ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/build/ubi/Dockerfile.nginxplus b/build/ubi/Dockerfile.nginxplus new file mode 100644 index 0000000000..8140c36d30 --- /dev/null +++ b/build/ubi/Dockerfile.nginxplus @@ -0,0 +1,79 @@ +# syntax=docker/dockerfile:1.18 +FROM scratch AS nginx-files + +# NGINX Plus repo and key files (must be provided at build time) +ADD --link --chown=101:1001 https://cs.nginx.com/static/files/plus-9.repo nginx-plus.repo +ADD --link --chown=101:1001 https://nginx.org/keys/nginx_signing.key nginx_signing.key +ADD --link --chown=101:1001 build/ubi/repos/agent.repo agent.repo + +FROM ghcr.io/nginx/dependencies/nginx-ubi:ubi9@sha256:01a32246761b9bbe47a6a29bcd8ca6e9b6e331b3bdfa372d8987b622276f7025 AS ubi9-packages + +FROM redhat/ubi9-minimal:9.6 AS ubi-nginx-plus + +ARG NGINX_PLUS_VERSION=R35 + +# renovate: datasource=github-tags depName=nginx/agent +ARG NGINX_AGENT_VERSION=v3.3.1 +ARG NJS_DIR +ARG NGINX_CONF_DIR +ARG BUILD_AGENT + +LABEL name="F5 NGINX Gateway Fabric NGINX Plus" \ + maintainer="kubernetes@nginx.com" \ + vendor="F5 NGINX" \ + summary="NGINX Gateway Fabric" \ + description="NGINX Gateway Fabric provides an implementation for the Gateway API using NGINX as the data plane." \ + org.nginx.ngf.image.build.agent="${BUILD_AGENT}" \ + io.k8s.description="NGINX Gateway Fabric provides an implementation for the Gateway API using NGINX as the data plane." \ + io.openshift.tags="nginx,gateway,kubernetes,openshift" + +COPY --link --chown=101:1001 LICENSE /licenses/ + +# Install NGINX Plus and modules +RUN --mount=type=bind,from=nginx-files,src=nginx-plus.repo,target=/etc/yum.repos.d/nginx-plus.repo \ + --mount=type=bind,from=nginx-files,src=agent.repo,target=/etc/yum.repos.d/agent.repo \ + --mount=type=bind,from=nginx-files,src=nginx_signing.key,target=/tmp/nginx_signing.key \ + --mount=type=bind,from=ubi9-packages,src=/,target=/ubi-bin/ \ + --mount=type=secret,id=nginx-repo.crt,dst=/etc/ssl/nginx/nginx-repo.crt,mode=0644 \ + --mount=type=secret,id=nginx-repo.key,dst=/etc/ssl/nginx/nginx-repo.key,mode=0644 \ + # Install shadow-utils for useradd + microdnf --nodocs install -y shadow-utils \ + && rpm --import /tmp/nginx_signing.key \ + # Install c-ares from the dependencies image (contains required libs) + && rpm -Uvh /ubi-bin/c-ares-*.rpm \ + # Create nginx user with consistent UID/GID + && groupadd -g 1001 nginx \ + && useradd -r -u 101 -g nginx -s /sbin/nologin -d /var/cache/nginx nginx \ + # Install NGINX Plus and modules (njs, otel) + && microdnf --nodocs install -y nginx-plus-${NGINX_PLUS_VERSION,,} \ + && microdnf --nodocs install -y nginx-plus-module-njs-${NGINX_PLUS_VERSION,,} nginx-plus-module-otel-${NGINX_PLUS_VERSION,,} \ + # Install nginx-agent + && microdnf --nodocs install -y nginx-agent-${NGINX_AGENT_VERSION#v}* \ + # Clean up + && microdnf remove -y shadow-utils \ + && microdnf clean all \ + && rm -rf /var/cache/yum + +# Configure directories and logging +RUN mkdir -p /var/run/nginx /usr/lib64/nginx/modules \ + # Forward request and error logs to docker log collector + && ln -sf /dev/stdout /var/log/nginx/access.log \ + && ln -sf /dev/stderr /var/log/nginx/error.log +# Copy default html files to a writable location +RUN mkdir -p /etc/nginx/html \ + && cp /usr/share/nginx/html/* /etc/nginx/html/ + +# Set proper permissions for nginx user +RUN chown -R 101:1001 /etc/nginx /var/cache/nginx + +# Copy configuration files and scripts +COPY build/entrypoint.sh /agent/entrypoint.sh +COPY ${NJS_DIR}/ /usr/lib64/nginx/modules/njs/ +COPY ${NGINX_CONF_DIR}/nginx.conf /etc/nginx/nginx.conf +COPY ${NGINX_CONF_DIR}/grpc-error-locations.conf /etc/nginx/grpc-error-locations.conf +COPY ${NGINX_CONF_DIR}/grpc-error-pages.conf /etc/nginx/grpc-error-pages.conf + +# Switch to non-root user +USER 101:1001 + +ENTRYPOINT ["/agent/entrypoint.sh"] diff --git a/build/ubi/repos/agent.repo b/build/ubi/repos/agent.repo new file mode 100644 index 0000000000..36665b874b --- /dev/null +++ b/build/ubi/repos/agent.repo @@ -0,0 +1,6 @@ +[agent] +name=agent repo +baseurl=https://packages.nginx.org/nginx-agent/centos/9/$basearch/ +gpgcheck=1 +enabled=1 +module_hotfixes=true diff --git a/build/ubi/repos/nginx.repo b/build/ubi/repos/nginx.repo new file mode 100644 index 0000000000..7c8e132faf --- /dev/null +++ b/build/ubi/repos/nginx.repo @@ -0,0 +1,6 @@ +[nginx] +name=nginx repo +baseurl=https://packages.nginx.org/nginx/mainline/centos/9/$basearch/ +gpgcheck=1 +enabled=1 +module_hotfixes=true diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml index 8fc4da400e..57c92e4692 100644 --- a/charts/nginx-gateway-fabric/templates/clusterrole.yaml +++ b/charts/nginx-gateway-fabric/templates/clusterrole.yaml @@ -7,15 +7,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml index ebf34b3d51..90e260460e 100644 --- a/deploy/azure/deploy.yaml +++ b/deploy/azure/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml index 3dd9d44a74..7dc53fe54a 100644 --- a/deploy/default/deploy.yaml +++ b/deploy/default/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml index dd08105202..4ea4257289 100644 --- a/deploy/experimental-nginx-plus/deploy.yaml +++ b/deploy/experimental-nginx-plus/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml index 4c837b8d90..6b9ff2594b 100644 --- a/deploy/experimental/deploy.yaml +++ b/deploy/experimental/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml index 21ded9d595..c7719908b2 100644 --- a/deploy/nginx-plus/deploy.yaml +++ b/deploy/nginx-plus/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml index fec6d7ca21..05bffae4c8 100644 --- a/deploy/nodeport/deploy.yaml +++ b/deploy/nodeport/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml index 61710631b0..cc837ac8d5 100644 --- a/deploy/openshift/deploy.yaml +++ b/deploy/openshift/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/snippets-filters-nginx-plus/deploy.yaml b/deploy/snippets-filters-nginx-plus/deploy.yaml index ce0c7cb908..9459a2673c 100644 --- a/deploy/snippets-filters-nginx-plus/deploy.yaml +++ b/deploy/snippets-filters-nginx-plus/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/deploy/snippets-filters/deploy.yaml b/deploy/snippets-filters/deploy.yaml index 1fe7cb5174..2500399f2e 100644 --- a/deploy/snippets-filters/deploy.yaml +++ b/deploy/snippets-filters/deploy.yaml @@ -55,15 +55,33 @@ metadata: rules: - apiGroups: - "" - - apps - - autoscaling resources: - secrets - configmaps - serviceaccounts - services + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - apps + resources: - deployments - daemonsets + verbs: + - create + - update + - delete + - list + - get + - watch +- apiGroups: + - autoscaling + resources: - horizontalpodautoscalers verbs: - create diff --git a/docs/developer/release-process.md b/docs/developer/release-process.md index 1a2d25a099..210417513b 100644 --- a/docs/developer/release-process.md +++ b/docs/developer/release-process.md @@ -58,6 +58,7 @@ To create a new release, follow these steps: - If the supported Gateway API minor version has changed since the last release, add a note to the release notes explaining if the previous version is no longer supported. - Merge the release PR once it has received all necessary approvals. 6. Once you are ready to release, run the [Production Release](https://github.com/nginx/nginx-gateway-fabric/actions/workflows/production-release.yml) workflow with the correct tag e.g. `v2.1.0`. (Note: It is also possible to do a dry run of the production release workflow for verification if required. This will not push the tag, images, and chart, and won't publish the release) +If this release includes an updated release of our [Operator](https://github.com/nginx/nginx-gateway-fabric/tree/main/operators), include the new version as well e.g. `v1.0.1` As a result, the CI/CD pipeline will: - Create and push the tag - Build NGF, NGINX and NGINX Plus container images with the release tag `X.Y.Z` and push them to the registries. diff --git a/examples/http-request-header-filter/headers.yaml b/examples/http-request-header-filter/headers.yaml index 4324c41e9f..d1d5f8388a 100644 --- a/examples/http-request-header-filter/headers.yaml +++ b/examples/http-request-header-filter/headers.yaml @@ -38,7 +38,7 @@ data: pid /var/run/nginx.pid; - load_module /usr/lib/nginx/modules/ngx_http_js_module.so; + load_module modules/ngx_http_js_module.so; events {} diff --git a/internal/controller/nginx/conf/nginx-plus.conf b/internal/controller/nginx/conf/nginx-plus.conf index f2b0ec0dc8..bcf0bfc613 100644 --- a/internal/controller/nginx/conf/nginx-plus.conf +++ b/internal/controller/nginx/conf/nginx-plus.conf @@ -1,4 +1,4 @@ -load_module /usr/lib/nginx/modules/ngx_http_js_module.so; +load_module modules/ngx_http_js_module.so; include /etc/nginx/main-includes/*.conf; worker_processes auto; @@ -12,7 +12,7 @@ events { http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; - js_import /usr/lib/nginx/modules/njs/httpmatches.js; + js_import modules/njs/httpmatches.js; default_type application/octet-stream; diff --git a/internal/controller/nginx/conf/nginx.conf b/internal/controller/nginx/conf/nginx.conf index 791994fdf8..46179b930c 100644 --- a/internal/controller/nginx/conf/nginx.conf +++ b/internal/controller/nginx/conf/nginx.conf @@ -1,4 +1,4 @@ -load_module /usr/lib/nginx/modules/ngx_http_js_module.so; +load_module modules/ngx_http_js_module.so; include /etc/nginx/main-includes/*.conf; worker_processes auto; @@ -12,7 +12,7 @@ events { http { include /etc/nginx/conf.d/*.conf; include /etc/nginx/mime.types; - js_import /usr/lib/nginx/modules/njs/httpmatches.js; + js_import modules/njs/httpmatches.js; default_type application/octet-stream; diff --git a/operators/Dockerfile b/operators/Dockerfile new file mode 100644 index 0000000000..61d0038b35 --- /dev/null +++ b/operators/Dockerfile @@ -0,0 +1,16 @@ +FROM quay.io/operator-framework/helm-operator:v1.41.1 + +COPY LICENSE /licenses/LICENSE + +LABEL name="nginx-gateway-fabric-operator" \ + vendor="F5 NGINX" \ + version="1.0.0" \ + release="1" \ + summary="NGINX Gateway Fabric Operator" \ + description="Helm-based operator for NGINX Gateway Fabric" + +ENV HOME=/opt/helm +COPY operators/watches.yaml ${HOME}/watches.yaml +COPY charts ${HOME}/helm-charts +COPY config ${HOME}/config +WORKDIR ${HOME} diff --git a/operators/Makefile b/operators/Makefile new file mode 100644 index 0000000000..8f5f9db9ff --- /dev/null +++ b/operators/Makefile @@ -0,0 +1,161 @@ +# VERSION defines the project version for the bundle. +# Update this value when you upgrade the version of the operator. +VERSION ?= 1.0.0 + +# renovate: datasource=github-tags depName=operator-framework/operator-sdk +OPERATOR_SDK_VERSION ?= v1.41.1 + +# renovate: datasource=github-tags depName=kubernetes-sigs/kustomize +KUSTOMIZE_VERSION ?= v5.6.0 + +IMAGE_TAG_BASE ?= nginx-gateway-fabric-operator + +IMG ?= $(IMAGE_TAG_BASE):v$(VERSION) + +## Bundle config + +BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION) + +CHANNELS ?= stable +DEFAULT_CHANNEL ?= stable + +ifneq ($(origin CHANNELS), undefined) +BUNDLE_CHANNELS := --channels=$(CHANNELS) +endif +ifneq ($(origin DEFAULT_CHANNEL), undefined) +BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL) +endif +BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL) + +BUNDLE_GEN_FLAGS ?= -q --overwrite --version $(VERSION) $(BUNDLE_METADATA_OPTS) + +USE_IMAGE_DIGESTS ?= false # USE_IMAGE_DIGESTS defines if images are resolved via tags or digests +ifeq ($(USE_IMAGE_DIGESTS), true) + BUNDLE_GEN_FLAGS += --use-image-digests +endif + +.PHONY: all +all: docker-build + +.PHONY: help +help: ## Display this help. + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + +##@ Build + +.PHONY: run +run: helm-operator ## Run against the configured Kubernetes cluster in ~/.kube/config + $(HELM_OPERATOR) run + +.PHONY: docker-build +docker-build: ## Build docker image with the manager. + docker build --platform linux/$(ARCH) -f Dockerfile -t ${IMG} .. + +.PHONY: docker-push +docker-push: ## Push docker image with the manager. + docker push ${IMG} + +.PHONY: docker-load +docker-load: ## Load docker image with the manager to kind. + kind load docker-image ${IMG} + +# PLATFORMS defines the target platforms for the manager image be build to provide support to multiple +# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). +PLATFORMS ?= linux/arm64,linux/amd64 +.PHONY: docker-buildx +docker-buildx: ## Build and push docker image for the manager for cross-platform support + - docker buildx create --name project-v3-builder + docker buildx use project-v3-builder + - docker buildx build --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile .. + - docker buildx rm project-v3-builder + +##@ Deployment + +.PHONY: install +install: kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl apply -f - + +.PHONY: uninstall +uninstall: kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/crd | kubectl delete -f - + +.PHONY: deploy +deploy: kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. + cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} + $(KUSTOMIZE) build config/default | kubectl apply -f - + +.PHONY: undeploy +undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. + $(KUSTOMIZE) build config/default | kubectl delete -f - + +OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +ARCH ?= $(shell uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') + +.PHONY: kustomize +KUSTOMIZE = $(shell pwd)/bin/kustomize +kustomize: ## Download kustomize locally if necessary. +ifeq (,$(wildcard $(KUSTOMIZE))) +ifeq (,$(shell which kustomize 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(KUSTOMIZE)) ;\ + curl -sSLo - https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/$(KUSTOMIZE_VERSION)/kustomize_$(KUSTOMIZE_VERSION)_$(OS)_$(ARCH).tar.gz | \ + tar xzf - -C bin/ ;\ + } +else +KUSTOMIZE = $(shell which kustomize) +endif +endif + +.PHONY: helm-operator +HELM_OPERATOR = $(shell pwd)/bin/helm-operator +helm-operator: ## Download helm-operator locally if necessary, preferring the $(pwd)/bin path over global if both exist. +ifeq (,$(wildcard $(HELM_OPERATOR))) +ifeq (,$(shell which helm-operator 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(HELM_OPERATOR)) ;\ + curl -sSLo $(HELM_OPERATOR) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/helm-operator_$(OS)_$(ARCH) ;\ + chmod +x $(HELM_OPERATOR) ;\ + } +else +HELM_OPERATOR = $(shell which helm-operator) +endif +endif + +.PHONY: operator-sdk +OPERATOR_SDK ?= $(LOCALBIN)/operator-sdk +operator-sdk: ## Download operator-sdk locally if necessary. +ifeq (,$(wildcard $(OPERATOR_SDK))) +ifeq (, $(shell which operator-sdk 2>/dev/null)) + @{ \ + set -e ;\ + mkdir -p $(dir $(OPERATOR_SDK)) ;\ + curl -sSLo $(OPERATOR_SDK) https://github.com/operator-framework/operator-sdk/releases/download/$(OPERATOR_SDK_VERSION)/operator-sdk_$(OS)_$(ARCH) ;\ + chmod +x $(OPERATOR_SDK) ;\ + } +else +OPERATOR_SDK = $(shell which operator-sdk) +endif +endif + +.PHONY: bundle +bundle: kustomize operator-sdk ## Generate bundle manifests and metadata, then validate generated files. + $(OPERATOR_SDK) generate kustomize manifests -q + cd config/manager && $(KUSTOMIZE) edit set image controller=$(IMG) + $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) + ./scripts/update-bundle.sh + $(OPERATOR_SDK) bundle validate ./bundle + +.PHONY: bundle-build +bundle-build: ## Build the bundle image. + docker build -f bundle.Dockerfile -t $(BUNDLE_IMG) . + +.PHONY: bundle-push +bundle-push: ## Push the bundle image. + $(MAKE) docker-push IMG=$(BUNDLE_IMG) + +.PHONY: bundle-release +bundle-release: ## Prepare the bundle manifests and create the bundle using the image digest. + ./scripts/update-ngf-img-version.sh + $(MAKE) bundle USE_IMAGE_DIGESTS=true diff --git a/operators/PROJECT b/operators/PROJECT new file mode 100644 index 0000000000..e960620c21 --- /dev/null +++ b/operators/PROJECT @@ -0,0 +1,20 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html +domain: nginx.org +layout: +- helm.sdk.operatorframework.io/v1 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} +projectName: nginx-gateway-fabric +resources: +- api: + crdVersion: v1 + namespaced: true + domain: nginx.org + group: gateway + kind: NginxGatewayFabric + version: v1alpha1 +version: "3" diff --git a/operators/README.md b/operators/README.md new file mode 100644 index 0000000000..6426bac8bd --- /dev/null +++ b/operators/README.md @@ -0,0 +1,135 @@ +# NGINX Gateway Fabric Operator + +A Helm-based Kubernetes operator for deploying and managing [NGINX Gateway Fabric](https://github.com/nginx/nginx-gateway-fabric), an implementation of the Gateway API using NGINX as the data plane. + +## Overview + +The NGINX Gateway Fabric Operator simplifies the deployment and lifecycle management of NGINX Gateway Fabric in Kubernetes and OpenShift environments. It leverages the official NGINX Gateway Fabric Helm charts to provide a declarative way to install, configure, and manage Gateway API implementations. + +## Features + +- **Declarative Configuration**: Manage NGINX Gateway Fabric through Kubernetes custom resources +- **Helm Chart Integration**: Uses official NGINX Gateway Fabric Helm charts for reliable deployments +- **OpenShift Compatible**: Certified for Red Hat OpenShift with proper SecurityContextConstraints +- **Full Feature Support**: Supports all NGINX Gateway Fabric configuration options including: + - NGINX Plus integration + - Experimental Gateway API features + - Multiple deployment modes (Deployment/DaemonSet) + +## Prerequisites + +- Kubernetes 1.25+ or OpenShift 4.19+ +- Operator Lifecycle Manager (OLM) installed +- Gateway API CRDs installed + +## Installation + +### OpenShift OperatorHub + +1. Navigate to OperatorHub in your OpenShift console +2. Search for "NGINX Gateway Fabric Operator" +3. Install the operator + +## Usage + +### Basic Installation + +Create a `NginxGatewayFabric` custom resource to deploy NGINX Gateway Fabric: + +```yaml +apiVersion: gateway.nginx.org/v1alpha1 +kind: NginxGatewayFabric +metadata: + name: nginx-gateway-fabric +spec: + nginxGateway: + replicas: 2 + gatewayClassName: nginx + nginx: + service: + type: LoadBalancer +``` + +See [the example here](config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml). + +## Configuration Reference + +The `NginxGatewayFabric` custom resource accepts the same configuration options as the NGINX Gateway Fabric Helm chart. + +For complete configuration options, see the [Helm Chart Documentation](https://github.com/nginx/nginx-gateway-fabric/tree/main/charts/nginx-gateway-fabric/README.md#configuration). + +## Development + +### Building and Testing the Operator Locally + +```bash +# Build the operator image. If building for deploying on a cluster with different architecture from your local machine, append ARCH= e.g. `ARCH=amd64` to the below command +make docker-build IMG=/nginx-gateway-fabric-operator: + +# Push the image +make docker-push IMG=/nginx-gateway-fabric-operator: + +# Optionally load the image if running on kind +make docker-load IMG=/nginx-gateway-fabric-operator: + +# Generate and push bundle (must be publicly accessible remote registry, e.g. quay.io) +make bundle-build bundle-push IMG=/nginx-gateway-fabric-operator: BUNDLE_IMG=/nginx-gateway-fabric-operator-bundle: + +# Install olm on local cluster if required (e.g. if running on kind) +operator-sdk olm install + +# Run your bundle image +operator-sdk run bundle /nginx-gateway-fabric-operator-bundle: + +# Deploy NGF operand (modify the manifest if required) +kubectl apply -f config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml + +# Deploy test application +kubectl apply -f ../examples/cafe-example/ + +# Run operator-sdk scorecard - optional +make bundle +operator-sdk scorecard bundle/ +``` + +### Releases + +Once NGF has released, we can prepare the Operator release using the published NGF images. + +The Operator image is built using the Helm Chart from the root directory, so those changes are kept in sync by running `make docker-build` from the release branch (to be automated). The Operator image can be published and certified at the same time as the UBI based NGF control plane and OSS data plane images (instructions to follow, to be automated). + +Once the images are certified and published, we can create the bundle for certification. This is mostly a scripted (note: to be automated) process. +However, there are a few items that need to be kept in sync manually: + +1. RBAC: + The Operator requires RBAC rules to include permissions for anything the NGF Helm chart + can deploy (e.g. Pods, ConfigMaps, Gateways, HPAs, etc), and all permissions that NGF + itself has permissions for (e.g. all the Gateway APIs etc). + + If the RBAC permissions either for or of the underlying Helm Chart changes, these need to be updated in [RBAC manifest](config/rbac/role.yaml). + + The next time `make bundle` is ran, these RBAC changes will be reflected in the resulting bundle manifests. + +2. Sample manifest: + The [example manifest](config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml) may need to be updated either to add new important fields, or to change existing entries. + +3. Operator version: + Update the VERSION in the Makefile to reflect the version of the Operator being released. + +When you are ready to release the bundle, run `make release-bundle`. This will update the NGF image version tags, and create the bundle manifests. + +To test the bundle locally, follow the `Building and Testing the Operator Locally` above. + +To submit the bundle for certification, follow TBD. + +## License + +This project is licensed under the Apache License 2.0. See [LICENSE](../LICENSE) for details. + +## Support + +- Documentation: [NGINX Gateway Fabric Docs](https://docs.nginx.com/nginx-gateway-fabric/) +- Issues: [GitHub Issues](https://github.com/nginx/nginx-gateway-fabric/issues) +- Community: [NGINX Community Forum](https://community.nginx.org/c/nginx-gateway-fabric) + +For commercial support, contact [F5 NGINX](https://www.f5.com/products/nginx). diff --git a/operators/bundle.Dockerfile b/operators/bundle.Dockerfile new file mode 100644 index 0000000000..59fbd69cd6 --- /dev/null +++ b/operators/bundle.Dockerfile @@ -0,0 +1,21 @@ +FROM scratch + +# Core bundle labels. +LABEL operators.operatorframework.io.bundle.mediatype.v1=registry+v1 +LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/ +LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/ +LABEL operators.operatorframework.io.bundle.package.v1=nginx-gateway-fabric +LABEL operators.operatorframework.io.bundle.channels.v1=stable +LABEL operators.operatorframework.io.bundle.channel.default.v1=stable +LABEL operators.operatorframework.io.metrics.builder=operator-sdk-v1.41.1 +LABEL operators.operatorframework.io.metrics.mediatype.v1=metrics+v1 +LABEL operators.operatorframework.io.metrics.project_layout=helm.sdk.operatorframework.io/v1 + +# Labels for testing. +LABEL operators.operatorframework.io.test.mediatype.v1=scorecard+v1 +LABEL operators.operatorframework.io.test.config.v1=tests/scorecard/ + +# Copy files to locations specified by labels. +COPY bundle/manifests /manifests/ +COPY bundle/metadata /metadata/ +COPY bundle/tests/scorecard /tests/scorecard/ diff --git a/operators/bundle/manifests/gateway.nginx.org_nginxgatewayfabrics.yaml b/operators/bundle/manifests/gateway.nginx.org_nginxgatewayfabrics.yaml new file mode 100644 index 0000000000..d031194f22 --- /dev/null +++ b/operators/bundle/manifests/gateway.nginx.org_nginxgatewayfabrics.yaml @@ -0,0 +1,104 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + creationTimestamp: null + name: nginxgatewayfabrics.gateway.nginx.org +spec: + group: gateway.nginx.org + names: + kind: NginxGatewayFabric + listKind: NginxGatewayFabricList + plural: nginxgatewayfabrics + singular: nginxgatewayfabric + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NginxGatewayFabric is the Schema for the nginxgatewayfabrics + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of NginxGatewayFabric and + contains Helm values for the NGF chart + properties: + certGenerator: + description: Certificate generator configuration + type: object + x-kubernetes-preserve-unknown-fields: true + clusterDomain: + description: The DNS cluster domain of your Kubernetes cluster + type: string + gateways: + description: List of Gateway objects to create + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + nginx: + description: Configuration for NGINX data plane deployments + type: object + x-kubernetes-preserve-unknown-fields: true + nginxGateway: + description: Configuration for the NGINX Gateway Fabric control plane + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + x-kubernetes-preserve-unknown-fields: true + status: + description: Status defines the observed state of NginxGatewayFabric + properties: + conditions: + description: Conditions represent the latest available observations + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + enum: + - "True" + - "False" + - Unknown + type: string + type: + type: string + type: object + type: array + phase: + description: Phase represents the current phase + enum: + - Pending + - Installing + - Ready + - Failed + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/operators/bundle/manifests/nginx-gateway-fabric-controller-manager-metrics-service_v1_service.yaml b/operators/bundle/manifests/nginx-gateway-fabric-controller-manager-metrics-service_v1_service.yaml new file mode 100644 index 0000000000..814eaf57d6 --- /dev/null +++ b/operators/bundle/manifests/nginx-gateway-fabric-controller-manager-metrics-service_v1_service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + creationTimestamp: null + labels: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: nginx-gateway-fabric + control-plane: controller-manager + name: nginx-gateway-fabric-controller-manager-metrics-service +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + app.kubernetes.io/name: nginx-gateway-fabric + control-plane: controller-manager +status: + loadBalancer: {} diff --git a/operators/bundle/manifests/nginx-gateway-fabric-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml b/operators/bundle/manifests/nginx-gateway-fabric-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml new file mode 100644 index 0000000000..a2e49c36ab --- /dev/null +++ b/operators/bundle/manifests/nginx-gateway-fabric-metrics-reader_rbac.authorization.k8s.io_v1_clusterrole.yaml @@ -0,0 +1,10 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: nginx-gateway-fabric-metrics-reader +rules: +- nonResourceURLs: + - /metrics + verbs: + - get diff --git a/operators/bundle/manifests/nginx-gateway-fabric.clusterserviceversion.yaml b/operators/bundle/manifests/nginx-gateway-fabric.clusterserviceversion.yaml new file mode 100644 index 0000000000..d953348c65 --- /dev/null +++ b/operators/bundle/manifests/nginx-gateway-fabric.clusterserviceversion.yaml @@ -0,0 +1,528 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: |- + [ + { + "apiVersion": "gateway.nginx.org/v1alpha1", + "kind": "NginxGatewayFabric", + "metadata": { + "name": "nginxgatewayfabric-sample" + }, + "spec": { + "certGenerator": { + "affinity": {}, + "agentTLSSecretName": "agent-tls", + "annotations": {}, + "nodeSelector": {}, + "overwrite": false, + "serverTLSSecretName": "server-tls", + "tolerations": [], + "topologySpreadConstraints": [], + "ttlSecondsAfterFinished": 30 + }, + "clusterDomain": "cluster.local", + "gateways": [], + "nginx": { + "autoscaling": { + "enable": false + }, + "config": {}, + "container": { + "hostPorts": [], + "lifecycle": {}, + "readinessProbe": {}, + "resources": {}, + "volumeMounts": [] + }, + "debug": false, + "image": { + "pullPolicy": "IfNotPresent", + "repository": "ghcr.io/nginx/nginx-gateway-fabric/nginx", + "tag": "2.2.0-ubi" + }, + "imagePullSecret": "", + "imagePullSecrets": [], + "kind": "deployment", + "nginxOneConsole": { + "dataplaneKeySecretName": "", + "endpointHost": "agent.connect.nginx.com", + "endpointPort": 443, + "skipVerify": false + }, + "patches": [], + "plus": false, + "pod": {}, + "replicas": 1, + "service": { + "externalTrafficPolicy": "Local", + "loadBalancerClass": "", + "loadBalancerIP": "", + "loadBalancerSourceRanges": [], + "nodePorts": [], + "patches": [], + "type": "LoadBalancer" + }, + "usage": { + "caSecretName": "", + "clientSSLSecretName": "", + "endpoint": "", + "enforceInitialReport": true, + "resolver": "", + "secretName": "nplus-license", + "skipVerify": false + } + }, + "nginxGateway": { + "affinity": {}, + "autoscaling": { + "enable": false + }, + "config": { + "logging": { + "level": "info" + } + }, + "configAnnotations": {}, + "extraVolumeMounts": [], + "extraVolumes": [], + "gatewayClassAnnotations": {}, + "gatewayClassName": "nginx", + "gatewayControllerName": "gateway.nginx.org/nginx-gateway-controller", + "gwAPIExperimentalFeatures": { + "enable": false + }, + "image": { + "pullPolicy": "IfNotPresent", + "repository": "ghcr.io/nginx/nginx-gateway-fabric", + "tag": "2.2.0-ubi" + }, + "kind": "deployment", + "labels": {}, + "leaderElection": { + "enable": true, + "lockName": "" + }, + "lifecycle": {}, + "metrics": { + "enable": true, + "port": 9113, + "secure": false + }, + "name": "", + "nodeSelector": {}, + "podAnnotations": {}, + "productTelemetry": { + "enable": true + }, + "readinessProbe": { + "enable": true, + "initialDelaySeconds": 3, + "port": 8081 + }, + "replicas": 1, + "resources": {}, + "service": { + "annotations": {}, + "labels": {} + }, + "serviceAccount": { + "annotations": {}, + "imagePullSecret": "", + "imagePullSecrets": [], + "name": "" + }, + "snippetsFilters": { + "enable": false + }, + "terminationGracePeriodSeconds": 30, + "tolerations": [], + "topologySpreadConstraints": [] + } + } + } + ] + capabilities: Basic Install + createdAt: "2025-09-25T13:06:47Z" + operators.operatorframework.io/builder: operator-sdk-v1.41.1 + operators.operatorframework.io/project_layout: helm.sdk.operatorframework.io/v1 + name: nginx-gateway-fabric.v1.0.0 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: + owned: + - kind: NginxGatewayFabric + name: nginxgatewayfabrics.gateway.nginx.org + version: v1alpha1 + resources: + - kind: Deployment + name: "" + version: v1 + - kind: Service + name: "" + version: v1 + - kind: ConfigMap + name: "" + version: v1 + - kind: Secret + name: "" + version: v1 + - kind: ServiceAccount + name: "" + version: v1 + - kind: ClusterRole + name: "" + version: v1 + - kind: ClusterRoleBinding + name: "" + version: v1 + specDescriptors: + - path: clusterDomain + displayName: Cluster Domain + description: The DNS cluster domain of your Kubernetes cluster + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:text + - path: nginxGateway + displayName: NGINX Gateway Configuration + description: Configuration for the NGINX Gateway Fabric control plane + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:NGINX Gateway + - path: nginx + displayName: NGINX Configuration + description: Configuration for NGINX data plane deployments + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:NGINX + - path: gateways + displayName: Gateways + description: List of Gateway objects to create + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Gateways + - path: certGenerator + displayName: Certificate Generator + description: Configuration for TLS certificate generation + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:fieldGroup:Certificate Generator + description: NGINX Gateway Fabric provides an implementation of the Gateway API using NGINX as the data plane + displayName: NGINX Gateway Fabric + icon: + - base64data: "" + mediatype: "" + install: + spec: + clusterPermissions: + - rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + - serviceaccounts + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + - referencegrants + - grpcroutes + - backendtlspolicies + - tlsroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + - gateways/status + - gatewayclasses/status + - grpcroutes/status + - backendtlspolicies/status + - tlsroutes/status + verbs: + - update + - apiGroups: + - gateway.nginx.org + resources: + - nginxgatewayfabrics + - nginxgatewayfabrics/status + - nginxgatewayfabrics/finalizers + - nginxgateways + - nginxgateways/status + - nginxgateways/finalizers + - nginxproxies + - nginxproxies/status + - nginxproxies/finalizers + - clientsettingspolicies + - observabilitypolicies + - upstreamsettingspolicies + - snippetsfilters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - gateway.nginx.org + resources: + - clientsettingspolicies/status + - observabilitypolicies/status + - upstreamsettingspolicies/status + - snippetsfilters/status + verbs: + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - delete + - get + - list + - patch + - update + - use + - watch + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create + serviceAccountName: nginx-gateway-fabric-controller-manager + deployments: + - label: + app.kubernetes.io/managed-by: kustomize + app.kubernetes.io/name: nginx-gateway-fabric + control-plane: controller-manager + name: nginx-gateway-fabric-controller-manager + spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: nginx-gateway-fabric + control-plane: controller-manager + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + app.kubernetes.io/name: nginx-gateway-fabric + control-plane: controller-manager + spec: + containers: + - args: + - --metrics-require-rbac + - --metrics-secure + - --metrics-bind-address=:8443 + - --leader-elect + - --leader-election-id=nginx-gateway-fabric + - --health-probe-bind-address=:8081 + image: nginx-gateway-fabric-operator:v1.0.0 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + name: manager + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + serviceAccountName: nginx-gateway-fabric-controller-manager + terminationGracePeriodSeconds: 10 + permissions: + - rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + serviceAccountName: nginx-gateway-fabric-controller-manager + strategy: deployment + installModes: + - supported: false + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - nginx + - gateway + - kubernetes + - networking + links: + - name: Nginx Gateway Fabric + url: https://nginx-gateway-fabric.domain + maintainers: + - email: kubernetes@nginx.com + name: F5NGINX + maturity: alpha + minKubeVersion: 1.25.0 + provider: + name: F5 NGINX + url: https://www.f5.com/go/product/welcome-to-nginx + version: 1.0.0 diff --git a/operators/bundle/metadata/annotations.yaml b/operators/bundle/metadata/annotations.yaml new file mode 100644 index 0000000000..b9684eacb3 --- /dev/null +++ b/operators/bundle/metadata/annotations.yaml @@ -0,0 +1,15 @@ +annotations: + # Core bundle annotations. + operators.operatorframework.io.bundle.mediatype.v1: registry+v1 + operators.operatorframework.io.bundle.manifests.v1: manifests/ + operators.operatorframework.io.bundle.metadata.v1: metadata/ + operators.operatorframework.io.bundle.package.v1: nginx-gateway-fabric + operators.operatorframework.io.bundle.channels.v1: stable + operators.operatorframework.io.bundle.channel.default.v1: stable + operators.operatorframework.io.metrics.builder: operator-sdk-v1.41.1 + operators.operatorframework.io.metrics.mediatype.v1: metrics+v1 + operators.operatorframework.io.metrics.project_layout: helm.sdk.operatorframework.io/v1 + + # Annotations for testing. + operators.operatorframework.io.test.mediatype.v1: scorecard+v1 + operators.operatorframework.io.test.config.v1: tests/scorecard/ diff --git a/operators/bundle/tests/scorecard/config.yaml b/operators/bundle/tests/scorecard/config.yaml new file mode 100644 index 0000000000..6ffe8227fa --- /dev/null +++ b/operators/bundle/tests/scorecard/config.yaml @@ -0,0 +1,70 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: + - entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: basic + test: basic-check-spec-test + storage: + spec: + mountPath: {} + - entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-bundle-validation-test + storage: + spec: + mountPath: {} + - entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-crds-have-validation-test + storage: + spec: + mountPath: {} + - entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-crds-have-resources-test + storage: + spec: + mountPath: {} + - entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-spec-descriptors-test + storage: + spec: + mountPath: {} + - entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-status-descriptors-test + storage: + spec: + mountPath: {} +storage: + spec: + mountPath: {} diff --git a/operators/config/crd/bases/gateway.nginx.org_nginxgatewayfabrics.yaml b/operators/config/crd/bases/gateway.nginx.org_nginxgatewayfabrics.yaml new file mode 100644 index 0000000000..4220ece70f --- /dev/null +++ b/operators/config/crd/bases/gateway.nginx.org_nginxgatewayfabrics.yaml @@ -0,0 +1,85 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: nginxgatewayfabrics.gateway.nginx.org +spec: + group: gateway.nginx.org + names: + kind: NginxGatewayFabric + listKind: NginxGatewayFabricList + plural: nginxgatewayfabrics + singular: nginxgatewayfabric + scope: Namespaced + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + description: NginxGatewayFabric is the Schema for the nginxgatewayfabrics API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: Spec defines the desired state of NginxGatewayFabric and contains Helm values for the NGF chart + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + clusterDomain: + description: The DNS cluster domain of your Kubernetes cluster + type: string + nginxGateway: + description: Configuration for the NGINX Gateway Fabric control plane + type: object + x-kubernetes-preserve-unknown-fields: true + nginx: + description: Configuration for NGINX data plane deployments + type: object + x-kubernetes-preserve-unknown-fields: true + gateways: + description: List of Gateway objects to create + type: array + items: + type: object + x-kubernetes-preserve-unknown-fields: true + certGenerator: + description: Certificate generator configuration + type: object + x-kubernetes-preserve-unknown-fields: true + status: + description: Status defines the observed state of NginxGatewayFabric + type: object + x-kubernetes-preserve-unknown-fields: true + properties: + conditions: + description: Conditions represent the latest available observations + type: array + items: + type: object + properties: + type: + type: string + status: + type: string + enum: ["True", "False", "Unknown"] + lastTransitionTime: + type: string + format: date-time + reason: + type: string + message: + type: string + phase: + description: Phase represents the current phase + type: string + enum: ["Pending", "Installing", "Ready", "Failed"] + subresources: + status: {} diff --git a/operators/config/crd/kustomization.yaml b/operators/config/crd/kustomization.yaml new file mode 100644 index 0000000000..19e8fdef34 --- /dev/null +++ b/operators/config/crd/kustomization.yaml @@ -0,0 +1,6 @@ +# This kustomization.yaml is not intended to be run by itself, +# since it depends on service name and namespace that are out of this kustomize package. +# It should be run by config/default +resources: +- bases/gateway.nginx.org_nginxgatewayfabrics.yaml +# +kubebuilder:scaffold:crdkustomizeresource diff --git a/operators/config/default/kustomization.yaml b/operators/config/default/kustomization.yaml new file mode 100644 index 0000000000..5f73b3bccd --- /dev/null +++ b/operators/config/default/kustomization.yaml @@ -0,0 +1,26 @@ +# Adds namespace to all resources. +namespace: nginx-gateway-fabric-system + +# Value of this field is prepended to the +# names of all resources, e.g. a deployment named +# "wordpress" becomes "alices-wordpress". +# Note that it should also match with the prefix (text before '-') of the namespace +# field above. +namePrefix: nginx-gateway-fabric- + +resources: +- ../crd +- ../rbac +- ../manager +# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. +# - ../prometheus +# [METRICS] Expose the controller manager metrics service. +- metrics_service.yaml + +# Uncomment the patches line if you enable Metrics +patches: +# [METRICS] The following patch will enable the metrics endpoint using HTTPS and the port :8443. +# More info: https://book.kubebuilder.io/reference/metrics +- path: manager_metrics_patch.yaml + target: + kind: Deployment diff --git a/operators/config/default/manager_metrics_patch.yaml b/operators/config/default/manager_metrics_patch.yaml new file mode 100644 index 0000000000..a3cb2f1865 --- /dev/null +++ b/operators/config/default/manager_metrics_patch.yaml @@ -0,0 +1,12 @@ +# This patch adds the args to allow exposing the metrics endpoint using HTTPS +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-bind-address=:8443 +# This patch adds the args to allow securing the metrics endpoint +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-secure +# This patch adds the args to allow RBAC-based authn/authz the metrics endpoint +- op: add + path: /spec/template/spec/containers/0/args/0 + value: --metrics-require-rbac diff --git a/operators/config/default/metrics_service.yaml b/operators/config/default/metrics_service.yaml new file mode 100644 index 0000000000..d428e3ff32 --- /dev/null +++ b/operators/config/default/metrics_service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-service + namespace: system +spec: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: 8443 + selector: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric diff --git a/operators/config/manager/kustomization.yaml b/operators/config/manager/kustomization.yaml new file mode 100644 index 0000000000..37bf8df3c3 --- /dev/null +++ b/operators/config/manager/kustomization.yaml @@ -0,0 +1,8 @@ +resources: +- manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: ghcr.io/nginx/nginx-gateway-fabric + newTag: v1.0.0 diff --git a/operators/config/manager/manager.yaml b/operators/config/manager/manager.yaml new file mode 100644 index 0000000000..6b20b20a5e --- /dev/null +++ b/operators/config/manager/manager.yaml @@ -0,0 +1,77 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system + labels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize +spec: + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + replicas: 1 + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: manager + labels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + spec: + securityContext: + # Projects are configured by default to adhere to the "restricted" Pod Security Standards. + # This ensures that deployments meet the highest security requirements for Kubernetes. + # For more details, see: https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + containers: + - args: + - --leader-elect + - --leader-election-id=nginx-gateway-fabric + - --health-probe-bind-address=:8081 + image: controller:latest + name: manager + ports: [] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - "ALL" + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + # TODO(user): Configure the resources accordingly based on the project requirements. + # More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 50m + memory: 64Mi + volumeMounts: [] + volumes: [] + serviceAccountName: controller-manager + terminationGracePeriodSeconds: 10 diff --git a/operators/config/manifests/bases/nginx-gateway-fabric.clusterserviceversion.yaml b/operators/config/manifests/bases/nginx-gateway-fabric.clusterserviceversion.yaml new file mode 100644 index 0000000000..278681ad81 --- /dev/null +++ b/operators/config/manifests/bases/nginx-gateway-fabric.clusterserviceversion.yaml @@ -0,0 +1,47 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + annotations: + alm-examples: '[]' + capabilities: Basic Install + name: nginx-gateway-fabric.v0.0.0 + namespace: placeholder +spec: + apiservicedefinitions: {} + customresourcedefinitions: {} + description: NGINX Gateway Fabric provides an implementation of the Gateway API + using NGINX as the data plane + displayName: NGINX Gateway Fabric + icon: + - base64data: "" + mediatype: "" + install: + spec: + deployments: null + strategy: "" + installModes: + - supported: false + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: true + type: AllNamespaces + keywords: + - nginx + - gateway + - kubernetes + - networking + links: + - name: Nginx Gateway Fabric + url: https://nginx-gateway-fabric.domain + maintainers: + - email: kubernetes@nginx.com + name: F5NGINX + maturity: alpha + minKubeVersion: 1.25.0 + provider: + name: F5 NGINX + url: https://www.f5.com/go/product/welcome-to-nginx + version: 0.0.0 diff --git a/operators/config/manifests/kustomization.yaml b/operators/config/manifests/kustomization.yaml new file mode 100644 index 0000000000..9107763129 --- /dev/null +++ b/operators/config/manifests/kustomization.yaml @@ -0,0 +1,7 @@ +# These resources constitute the fully configured set of manifests +# used to generate the 'manifests/' directory in a bundle. +resources: +- bases/nginx-gateway-fabric.clusterserviceversion.yaml +- ../default +- ../samples +- ../scorecard diff --git a/operators/config/network-policy/allow-metrics-traffic.yaml b/operators/config/network-policy/allow-metrics-traffic.yaml new file mode 100644 index 0000000000..44e662426a --- /dev/null +++ b/operators/config/network-policy/allow-metrics-traffic.yaml @@ -0,0 +1,27 @@ +# This NetworkPolicy allows ingress traffic +# with Pods running on namespaces labeled with 'metrics: enabled'. Only Pods on those +# namespaces are able to gather data from the metrics endpoint. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + labels: + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: allow-metrics-traffic + namespace: system +spec: + podSelector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + policyTypes: + - Ingress + ingress: + # This allows ingress traffic from any namespace with the label metrics: enabled + - from: + - namespaceSelector: + matchLabels: + metrics: enabled # Only from namespaces with this label + ports: + - port: 8443 + protocol: TCP diff --git a/operators/config/network-policy/kustomization.yaml b/operators/config/network-policy/kustomization.yaml new file mode 100644 index 0000000000..ec0fb5e57d --- /dev/null +++ b/operators/config/network-policy/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- allow-metrics-traffic.yaml diff --git a/operators/config/prometheus/kustomization.yaml b/operators/config/prometheus/kustomization.yaml new file mode 100644 index 0000000000..ed137168a1 --- /dev/null +++ b/operators/config/prometheus/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- monitor.yaml diff --git a/operators/config/prometheus/monitor.yaml b/operators/config/prometheus/monitor.yaml new file mode 100644 index 0000000000..6d9d59f74a --- /dev/null +++ b/operators/config/prometheus/monitor.yaml @@ -0,0 +1,22 @@ +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + labels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: controller-manager-metrics-monitor + namespace: system +spec: + endpoints: + - path: /metrics + port: https # Ensure this is the name of the port that exposes HTTPS metrics + scheme: https + bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token + tlsConfig: + insecureSkipVerify: true + selector: + matchLabels: + control-plane: controller-manager + app.kubernetes.io/name: nginx-gateway-fabric diff --git a/operators/config/rbac/kustomization.yaml b/operators/config/rbac/kustomization.yaml new file mode 100644 index 0000000000..5619aa0094 --- /dev/null +++ b/operators/config/rbac/kustomization.yaml @@ -0,0 +1,20 @@ +resources: +# All RBAC will be applied under this service account in +# the deployment namespace. You may comment out this resource +# if your manager will use a service account that exists at +# runtime. Be sure to update RoleBinding and ClusterRoleBinding +# subjects if changing service account names. +- service_account.yaml +- role.yaml +- role_binding.yaml +- leader_election_role.yaml +- leader_election_role_binding.yaml +# The following RBAC configurations are used to protect +# the metrics endpoint with authn/authz. These configurations +# ensure that only authorized users and service accounts +# can access the metrics endpoint. Comment the following +# permissions if you want to disable this protection. +# More info: https://book.kubebuilder.io/reference/metrics.html +- metrics_auth_role.yaml +- metrics_auth_role_binding.yaml +- metrics_reader_role.yaml diff --git a/operators/config/rbac/leader_election_role.yaml b/operators/config/rbac/leader_election_role.yaml new file mode 100644 index 0000000000..f45d6da4c7 --- /dev/null +++ b/operators/config/rbac/leader_election_role.yaml @@ -0,0 +1,40 @@ +# permissions to do leader election. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: leader-election-role +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/operators/config/rbac/leader_election_role_binding.yaml b/operators/config/rbac/leader_election_role_binding.yaml new file mode 100644 index 0000000000..d4b412edae --- /dev/null +++ b/operators/config/rbac/leader_election_role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: leader-election-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: leader-election-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/operators/config/rbac/metrics_auth_role.yaml b/operators/config/rbac/metrics_auth_role.yaml new file mode 100644 index 0000000000..32d2e4ec6b --- /dev/null +++ b/operators/config/rbac/metrics_auth_role.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-auth-role +rules: +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create diff --git a/operators/config/rbac/metrics_auth_role_binding.yaml b/operators/config/rbac/metrics_auth_role_binding.yaml new file mode 100644 index 0000000000..e775d67ff0 --- /dev/null +++ b/operators/config/rbac/metrics_auth_role_binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metrics-auth-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metrics-auth-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/operators/config/rbac/metrics_reader_role.yaml b/operators/config/rbac/metrics_reader_role.yaml new file mode 100644 index 0000000000..51a75db47a --- /dev/null +++ b/operators/config/rbac/metrics_reader_role.yaml @@ -0,0 +1,9 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metrics-reader +rules: +- nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/operators/config/rbac/role.yaml b/operators/config/rbac/role.yaml new file mode 100644 index 0000000000..5dcd5a0937 --- /dev/null +++ b/operators/config/rbac/role.yaml @@ -0,0 +1,185 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manager-role +rules: +- apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - services/finalizers + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + - serviceaccounts + verbs: + - "*" +- apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - batch + resources: + - jobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + - roles + - rolebindings + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + - httproutes + - referencegrants + - grpcroutes + - backendtlspolicies + - tlsroutes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - httproutes/status + - gateways/status + - gatewayclasses/status + - grpcroutes/status + - backendtlspolicies/status + - tlsroutes/status + verbs: + - update +- apiGroups: + - gateway.nginx.org + resources: + - nginxgatewayfabrics + - nginxgatewayfabrics/status + - nginxgatewayfabrics/finalizers + - nginxgateways + - nginxgateways/status + - nginxgateways/finalizers + - nginxproxies + - nginxproxies/status + - nginxproxies/finalizers + - clientsettingspolicies + - observabilitypolicies + - upstreamsettingspolicies + - snippetsfilters + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - gateway.nginx.org + resources: + - clientsettingspolicies/status + - observabilitypolicies/status + - upstreamsettingspolicies/status + - snippetsfilters/status + verbs: + - update +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - update +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - delete + - get + - list + - patch + - update + - use + - watch + +# +kubebuilder:scaffold:rules diff --git a/operators/config/rbac/role_binding.yaml b/operators/config/rbac/role_binding.yaml new file mode 100644 index 0000000000..e718c442fe --- /dev/null +++ b/operators/config/rbac/role_binding.yaml @@ -0,0 +1,15 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: manager-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: manager-role +subjects: +- kind: ServiceAccount + name: controller-manager + namespace: system diff --git a/operators/config/rbac/service_account.yaml b/operators/config/rbac/service_account.yaml new file mode 100644 index 0000000000..1e60df5111 --- /dev/null +++ b/operators/config/rbac/service_account.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: nginx-gateway-fabric + app.kubernetes.io/managed-by: kustomize + name: controller-manager + namespace: system diff --git a/operators/config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml b/operators/config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml new file mode 100644 index 0000000000..c52e22e49d --- /dev/null +++ b/operators/config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml @@ -0,0 +1,114 @@ +apiVersion: gateway.nginx.org/v1alpha1 +kind: NginxGatewayFabric +metadata: + name: nginxgatewayfabric-sample +spec: + # Default values copied from /helm-charts/nginx-gateway-fabric/values.yaml + certGenerator: + affinity: {} + agentTLSSecretName: agent-tls + annotations: {} + nodeSelector: {} + overwrite: false + serverTLSSecretName: server-tls + tolerations: [] + topologySpreadConstraints: [] + ttlSecondsAfterFinished: 30 + clusterDomain: cluster.local + gateways: [] + nginx: + autoscaling: + enable: false + config: {} + container: + hostPorts: [] + lifecycle: {} + readinessProbe: {} + resources: {} + volumeMounts: [] + debug: false + image: + pullPolicy: IfNotPresent + repository: ghcr.io/nginx/nginx-gateway-fabric/nginx + tag: 2.2.0-ubi + imagePullSecret: "" + imagePullSecrets: [] + kind: deployment + nginxOneConsole: + dataplaneKeySecretName: "" + endpointHost: agent.connect.nginx.com + endpointPort: 443 + skipVerify: false + patches: [] + plus: false + pod: {} + replicas: 1 + service: + externalTrafficPolicy: Local + loadBalancerClass: "" + loadBalancerIP: "" + loadBalancerSourceRanges: [] + nodePorts: [] + patches: [] + type: LoadBalancer + usage: + caSecretName: "" + clientSSLSecretName: "" + endpoint: "" + enforceInitialReport: true + resolver: "" + secretName: nplus-license + skipVerify: false + nginxGateway: + affinity: {} + autoscaling: + enable: false + config: + logging: + level: info + configAnnotations: {} + extraVolumeMounts: [] + extraVolumes: [] + gatewayClassAnnotations: {} + gatewayClassName: nginx + gatewayControllerName: gateway.nginx.org/nginx-gateway-controller + gwAPIExperimentalFeatures: + enable: false + image: + pullPolicy: IfNotPresent + repository: ghcr.io/nginx/nginx-gateway-fabric + tag: 2.2.0-ubi + kind: deployment + labels: {} + leaderElection: + enable: true + lockName: "" + lifecycle: {} + metrics: + enable: true + port: 9113 + secure: false + name: "" + nodeSelector: {} + podAnnotations: {} + productTelemetry: + enable: true + readinessProbe: + enable: true + initialDelaySeconds: 3 + port: 8081 + replicas: 1 + resources: {} + service: + annotations: {} + labels: {} + serviceAccount: + annotations: {} + imagePullSecret: "" + imagePullSecrets: [] + name: "" + snippetsFilters: + enable: false + terminationGracePeriodSeconds: 30 + tolerations: [] + topologySpreadConstraints: [] diff --git a/operators/config/samples/kustomization.yaml b/operators/config/samples/kustomization.yaml new file mode 100644 index 0000000000..f381eefd33 --- /dev/null +++ b/operators/config/samples/kustomization.yaml @@ -0,0 +1,4 @@ +## Append samples of your project ## +resources: +- gateway_v1alpha1_nginxgatewayfabric.yaml +# +kubebuilder:scaffold:manifestskustomizesamples diff --git a/operators/config/scorecard/bases/config.yaml b/operators/config/scorecard/bases/config.yaml new file mode 100644 index 0000000000..c77047841e --- /dev/null +++ b/operators/config/scorecard/bases/config.yaml @@ -0,0 +1,7 @@ +apiVersion: scorecard.operatorframework.io/v1alpha3 +kind: Configuration +metadata: + name: config +stages: +- parallel: true + tests: [] diff --git a/operators/config/scorecard/kustomization.yaml b/operators/config/scorecard/kustomization.yaml new file mode 100644 index 0000000000..54e8aa5075 --- /dev/null +++ b/operators/config/scorecard/kustomization.yaml @@ -0,0 +1,18 @@ +resources: +- bases/config.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +patches: +- path: patches/basic.config.yaml + target: + group: scorecard.operatorframework.io + kind: Configuration + name: config + version: v1alpha3 +- path: patches/olm.config.yaml + target: + group: scorecard.operatorframework.io + kind: Configuration + name: config + version: v1alpha3 +# +kubebuilder:scaffold:patches diff --git a/operators/config/scorecard/patches/basic.config.yaml b/operators/config/scorecard/patches/basic.config.yaml new file mode 100644 index 0000000000..8237b70d80 --- /dev/null +++ b/operators/config/scorecard/patches/basic.config.yaml @@ -0,0 +1,10 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - basic-check-spec + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: basic + test: basic-check-spec-test diff --git a/operators/config/scorecard/patches/olm.config.yaml b/operators/config/scorecard/patches/olm.config.yaml new file mode 100644 index 0000000000..416660a77e --- /dev/null +++ b/operators/config/scorecard/patches/olm.config.yaml @@ -0,0 +1,50 @@ +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-bundle-validation + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-bundle-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-validation + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-crds-have-validation-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-crds-have-resources + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-crds-have-resources-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-spec-descriptors + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-spec-descriptors-test +- op: add + path: /stages/0/tests/- + value: + entrypoint: + - scorecard-test + - olm-status-descriptors + image: quay.io/operator-framework/scorecard-test:v1.41.1 + labels: + suite: olm + test: olm-status-descriptors-test diff --git a/operators/scripts/update-bundle.sh b/operators/scripts/update-bundle.sh new file mode 100755 index 0000000000..40336d32b9 --- /dev/null +++ b/operators/scripts/update-bundle.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# update-bundle.sh - Run after 'make bundle' to add missing scorecard fields + +CSV_FILE="bundle/manifests/nginx-gateway-fabric.clusterserviceversion.yaml" + +# Check if CSV file exists +if [ ! -f "$CSV_FILE" ]; then + echo "Error: CSV file not found at $CSV_FILE" + exit 1 +fi + +echo "Adding resources and specDescriptors to $CSV_FILE..." + +# Use yq to add the resources and specDescriptors +yq eval ' +.spec.customresourcedefinitions.owned[0].resources = [ + {"kind": "Deployment", "name": "", "version": "v1"}, + {"kind": "Service", "name": "", "version": "v1"}, + {"kind": "ConfigMap", "name": "", "version": "v1"}, + {"kind": "Secret", "name": "", "version": "v1"}, + {"kind": "ServiceAccount", "name": "", "version": "v1"}, + {"kind": "ClusterRole", "name": "", "version": "v1"}, + {"kind": "ClusterRoleBinding", "name": "", "version": "v1"} +] | +.spec.customresourcedefinitions.owned[0].specDescriptors = [ + {"path": "clusterDomain", "displayName": "Cluster Domain", "description": "The DNS cluster domain of your Kubernetes cluster", "x-descriptors": ["urn:alm:descriptor:com.tectonic.ui:text"]}, + {"path": "nginxGateway", "displayName": "NGINX Gateway Configuration", "description": "Configuration for the NGINX Gateway Fabric control plane", "x-descriptors": ["urn:alm:descriptor:com.tectonic.ui:fieldGroup:NGINX Gateway"]}, + {"path": "nginx", "displayName": "NGINX Configuration", "description": "Configuration for NGINX data plane deployments", "x-descriptors": ["urn:alm:descriptor:com.tectonic.ui:fieldGroup:NGINX"]}, + {"path": "gateways", "displayName": "Gateways", "description": "List of Gateway objects to create", "x-descriptors": ["urn:alm:descriptor:com.tectonic.ui:fieldGroup:Gateways"]}, + {"path": "certGenerator", "displayName": "Certificate Generator", "description": "Configuration for TLS certificate generation", "x-descriptors": ["urn:alm:descriptor:com.tectonic.ui:fieldGroup:Certificate Generator"]} +] +' -i "$CSV_FILE" + +echo "Bundle updates applied successfully!" diff --git a/operators/scripts/update-ngf-img-version.sh b/operators/scripts/update-ngf-img-version.sh new file mode 100755 index 0000000000..97abacd932 --- /dev/null +++ b/operators/scripts/update-ngf-img-version.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +# Get NGF version from Chart.yaml +NGF_VERSION=$(grep "^appVersion:" ../charts/nginx-gateway-fabric/Chart.yaml | sed 's/appVersion: *//g' | tr -d '"') + +echo "Using NGF version: $NGF_VERSION" + +# Update sample file image tags +sed -i '' "s/tag: .*/tag: \"$NGF_VERSION\"/" config/samples/gateway_v1alpha1_nginxgatewayfabric.yaml + +echo "Done. Run 'make bundle' next." diff --git a/operators/watches.yaml b/operators/watches.yaml new file mode 100644 index 0000000000..77673bfc89 --- /dev/null +++ b/operators/watches.yaml @@ -0,0 +1,6 @@ +# Use the 'create api' subcommand to add watches to this file. +- group: gateway.nginx.org + version: v1alpha1 + kind: NginxGatewayFabric + chart: helm-charts/nginx-gateway-fabric +# +kubebuilder:scaffold:watch diff --git a/tests/Makefile b/tests/Makefile index ec0fe2edb5..79ed6e300b 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -13,9 +13,11 @@ NGF_VERSION ?= edge## NGF version to be tested PULL_POLICY = Never## Pull policy for the images NGINX_CONF_DIR = internal/controller/nginx/conf SUPPORTED_EXTENDED_FEATURES = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,GatewayAddressEmpty,HTTPRouteResponseHeaderModification,HTTPRoutePathRedirect,GatewayHTTPListenerIsolation,GatewayInfrastructurePropagation,HTTPRouteRequestMirror,HTTPRouteRequestMultipleMirrors,HTTPRouteRequestPercentageMirror,HTTPRouteBackendProtocolWebSocket,HTTPRouteParentRefPort,HTTPRouteDestinationPortMatching,GatewayStaticAddresses +SUPPORTED_EXTENDED_FEATURES_OPENSHIFT = HTTPRouteQueryParamMatching,HTTPRouteMethodMatching,HTTPRoutePortRedirect,HTTPRouteSchemeRedirect,HTTPRouteHostRewrite,HTTPRoutePathRewrite,GatewayPort8080,GatewayAddressEmpty,HTTPRouteResponseHeaderModification,HTTPRoutePathRedirect,GatewayHTTPListenerIsolation,GatewayInfrastructurePropagation,HTTPRouteRequestMirror,HTTPRouteRequestMultipleMirrors,HTTPRouteRequestPercentageMirror,HTTPRouteBackendProtocolWebSocket,HTTPRouteParentRefPort,HTTPRouteDestinationPortMatching STANDARD_CONFORMANCE_PROFILES = GATEWAY-HTTP,GATEWAY-GRPC EXPERIMENTAL_CONFORMANCE_PROFILES = GATEWAY-TLS CONFORMANCE_PROFILES = $(STANDARD_CONFORMANCE_PROFILES) # by default we use the standard conformance profiles. If experimental is enabled we override this and add the experimental profiles. +SKIP_TESTS_OPENSHIFT = HTTPRouteServiceTypes # Doesn't work on OpenShift due to security restrictions SKIP_TESTS = CEL_TEST_TARGET = @@ -40,7 +42,7 @@ update-go-modules: ## Update the gateway-api go modules to latest main version .PHONY: build-test-runner-image build-test-runner-image: ## Build conformance test runner image - docker build -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile . + docker build --platform $(GOOS)/$(GOARCH) -t $(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) -f conformance/Dockerfile . .PHONY: build-crossplane-image build-crossplane-image: ## Build the crossplane image @@ -58,6 +60,26 @@ run-conformance-tests: ## Run conformance tests --report-output=output.txt; cat output.txt" | tee output.txt ./scripts/check-pod-exit-code.sh sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml + grpc_core_result=`yq '.profiles[0].core.result' conformance-profile.yaml`; \ + http_core_result=`yq '.profiles[1].core.result' conformance-profile.yaml`; \ + http_extended_result=`yq '.profiles[1].extended.result' conformance-profile.yaml`; \ + if [ "$$grpc_core_result" != "failure" ] && [ "$$http_core_result" != "failure" ] && [ "$$http_extended_result" != "failure" ] ; then \ + exit 0; \ + else \ + exit 2; \ + fi + +.PHONY: run-conformance-tests-openshift +run-conformance-tests-openshift: ## Run conformance tests on OpenShift (skips tests incompatible with OpenShift) + kubectl apply -f conformance/conformance-rbac.yaml + kubectl run -i conformance \ + --image=$(CONFORMANCE_PREFIX):$(CONFORMANCE_TAG) --image-pull-policy=Always \ + --overrides='{ "spec": { "serviceAccountName": "conformance" } }' \ + --restart=Never -- sh -c "go test -v . -tags conformance,experimental -args --gateway-class=$(GATEWAY_CLASS) \ + --supported-features=$(SUPPORTED_EXTENDED_FEATURES_OPENSHIFT) --version=$(NGF_VERSION) --skip-tests=$(SKIP_TESTS_OPENSHIFT) --conformance-profiles=$(CONFORMANCE_PROFILES) \ + --service-type=$(GW_SERVICE_TYPE) --report-output=output.txt; cat output.txt" | tee output.txt + ./scripts/check-pod-exit-code.sh + sed -e '1,/CONFORMANCE PROFILE/d' output.txt > conformance-profile.yaml rm output.txt grpc_core_result=`yq '.profiles[0].core.result' conformance-profile.yaml`; \ http_core_result=`yq '.profiles[1].core.result' conformance-profile.yaml`; \ diff --git a/tests/OPENSHIFT_CONFORMANCE.md b/tests/OPENSHIFT_CONFORMANCE.md new file mode 100644 index 0000000000..a3e5c2e4b5 --- /dev/null +++ b/tests/OPENSHIFT_CONFORMANCE.md @@ -0,0 +1,150 @@ +# Running Gateway API Conformance Tests on OpenShift + +This document describes the steps required to run Gateway API conformance tests on an OpenShift cluster. + +## Prerequisites + +- Access to an OpenShift cluster +- `oc` CLI tool installed and configured +- `kubectl` configured to access your OpenShift cluster +- Docker/Podman for building images +- Access to a container registry (e.g., quay.io) +- NGF should be preinstalled on the cluster before running the tests. You can install using the Operator or Helm. +**Note** : + - the NGINX service type needs to be set to `ClusterIP` + - the NGINX image referenced in the `NginxProxy` resource needs to be accessible to the cluster + +## Overview + +OpenShift has stricter security constraints than standard Kubernetes, requiring additional configuration to run the Gateway API conformance test suite. + +## Step 1: Check Gateway API Version + +OpenShift ships with Gateway API CRDs pre-installed. To find out which version is installed, run the following command: + + ```bash + kubectl get crd gateways.gateway.networking.k8s.io -o jsonpath='{.metadata.annotations.gateway\.networking\.k8s\.io/bundle-version}' + ``` + +### Updating NGF to Match OpenShift's Gateway API Version + +To run conformance tests that match the exact Gateway API version on OpenShift: + +1. Update Go modules: + + ```bash + # Update parent module + go get sigs.k8s.io/gateway-api@ + go mod tidy + + # Update tests module + cd tests + go get sigs.k8s.io/gateway-api@ + go mod tidy + cd .. + ``` + + **Important:** Due to the `replace` directive in `tests/go.mod`, you must update both the parent and tests modules for the version change to take effect. + +2. Update test configuration to remove features not available in the OCP-installed Gateway API version. + +For **Gateway API v1.2.1**, you must update tests/conformance/conformance_test.go to eliminate references to v1beta1.GatewayStaticAddresses. This field was only introduced in Gateway API v1.3.0, and leaving it in place will cause the test to fail to compile in a v1.2.1 environment. + +**Note:** This is separate from `SUPPORTED_EXTENDED_FEATURES_OPENSHIFT` in the Makefile, which controls which features are tested. This change is required because the conformance test code itself references v1.3.0+ features that don't exist in v1.2.1. + +## Step 2: Build and Push Conformance Test Image + +OpenShift typically runs on amd64 architecture. If you are building images from an arm64 machine, make sure to specify the target platform so the image is built for the correct architecture + +1. Build the conformance test runner image for amd64: + + ```bash + make -C tests build-test-runner-image GOARCH=amd64 CONFORMANCE_PREFIX=//conformance-test-runner CONFORMANCE_TAG= + ``` + +2. Push the image to your registry: + + ```bash + docker push //conformance-test-runner: + ``` + +## Step 3: Configure Security Context Constraints (SCC) + +OpenShift requires explicit permissions for pods to run with elevated privileges. To apply SCC permissions to allow coredns and other infrastructure pods, run: + + ```bash + oc adm policy add-scc-to-group anyuid system:serviceaccounts:gateway-conformance-infra + ``` + +**Note:** These permissions persist even if the namespace is deleted and recreated during test runs. + +## Step 4: Run Conformance Tests + +### Using the Makefile (Recommended) + +Run the OpenShift-specific conformance test target: + +```bash +make -C tests run-conformance-tests-openshift \ + CONFORMANCE_PREFIX=quay.io/your-org/conformance-test-runner \ + CONFORMANCE_TAG= \ +``` + +This target: + +- Applies the RBAC configuration +- Runs only the extended features supported on the GatewayAPIs shipped with OpenShift +- Skips `HTTPRouteServiceTypes` test (incompatible with OpenShift) +- Pulls the image from your registry + +## Step 5: Known Test Failures on OpenShift + +### HTTPRouteServiceTypes + +This test fails on OpenShift due to security restrictions on EndpointSlice creation: + +```text +endpointslices.discovery.k8s.io "manual-endpointslices-ip4" is forbidden: +endpoint address 10.x.x.x is not allowed +``` + +**Solution:** Skip this test using `--skip-tests=HTTPRouteServiceTypes` + +This is expected behavior - OpenShift validates that endpoint IPs belong to approved ranges, and the conformance test tries to create EndpointSlices with arbitrary IPs. + +## Cleanup + +```bash +kubectl delete pod conformance +kubectl delete -f tests/conformance/conformance-rbac.yaml +``` + +## Troubleshooting + +### coredns pod fails with "Operation not permitted" + +**Cause:** Missing SCC permissions + +**Solution:** Apply the anyuid SCC as described in Step 3 + +### DNS resolution failures for LoadBalancer services + +**Cause:** OpenShift cluster DNS cannot resolve external ELB/LoadBalancer hostnames + +**Solution:** Use `GW_SERVICE_TYPE=ClusterIP` + +### Architecture mismatch errors ("Exec format error") + +**Cause:** Image built for wrong architecture (e.g., arm64 instead of amd64) + +**Solution:** Rebuild with `GOARCH=amd64` as described in Step 3 + +## Summary + +The key differences when running conformance tests on OpenShift vs. standard Kubernetes: + +1. **SCC Permissions:** Required for coredns and infrastructure pods +2. **Service Type:** Must use `ClusterIP` to avoid DNS issues +3. **Architecture:** Explicit amd64 build required when building from arm64 machines +4. **Test Skips:** HTTPRouteServiceTypes must be skipped due to EndpointSlice restrictions +5. **Image Registry:** Images must be pushed to a registry accessible by OpenShift diff --git a/tests/README.md b/tests/README.md index 883bc595bd..a1d55b79f2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -56,7 +56,7 @@ All the commands below are executed from the `tests` directory. You can see all ### Step 1 - Create a Kubernetes cluster -**Important**: Functional/conformance tests can only be run on a `kind` cluster. NFR tests can only be run on a GKE cluster. +**Important**: Functional tests can only be run on a `kind` cluster. Conformance tests can be run on `kind` or OpenShift clusters (see [OPENSHIFT_CONFORMANCE.md](OPENSHIFT_CONFORMANCE.md) for OpenShift instructions). NFR tests can only be run on a GKE cluster. To create a local `kind` cluster: