From c863c1a8b0933c05edf8085441dab9e5eb062446 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Tue, 7 Nov 2023 16:54:01 +0000 Subject: [PATCH 01/17] temp merge of all changes --- README.md | 8 +- docs/content/app-protect-dos/installation.md | 85 -- .../configuration/configuration-examples.md | 4 +- .../command-line-arguments.md | 197 ++-- .../configmap-resource.md | 18 +- .../global-configuration/custom-templates.md | 2 +- .../globalconfiguration-resource.md | 25 +- .../handling-host-and-listener-collisions.md | 6 +- ...advanced-configuration-with-annotations.md | 40 +- .../ingress-resources/basic-configuration.md | 4 +- .../cross-namespace-configuration.md | 4 +- .../ingress-resources/custom-annotations.md | 15 +- docs/content/configuration/policy-resource.md | 6 +- docs/content/configuration/security.md | 2 +- .../configuration/transportserver-resource.md | 14 +- ...server-and-virtualserverroute-resources.md | 7 +- docs/content/f5-ingresslink.md | 94 -- docs/content/glossary.md | 35 + docs/content/includes/index.md | 3 + .../installation/create-common-resources.md | 29 + .../installation/manifests/daemonset.md | 21 + .../installation/manifests/deployment.md | 21 + .../manifests/verify-pods-are-running.md | 9 + docs/content/includes/rbac/set-up-rbac.md | 33 + docs/content/installation/_index.md | 2 +- .../building-ingress-controller-image.md | 128 --- .../building-nginx-ingress-controller.md | 194 ++++ docs/content/installation/ingress-nginx.md | 552 +++++++++++ .../installation/installation-with-helm.md | 489 ---------- .../installation-with-manifests.md | 312 ------ .../installation-with-operator.md | 69 -- .../installation/installing-nic/_index.md | 5 + .../installing-nic/installation-with-helm.md | 436 +++++++++ .../installation-with-manifests.md | 228 +++++ .../installation-with-operator.md | 80 ++ .../installation/integrations/_index.md | 5 + .../integrations}/app-protect-dos/_index.md | 6 +- .../app-protect-dos/configuration.md | 11 +- .../app-protect-dos/dos-protected.md | 4 +- .../app-protect-dos/installation.md | 189 ++++ .../integrations}/app-protect-waf/_index.md | 4 +- .../app-protect-waf/configuration.md | 34 +- .../app-protect-waf/installation.md | 24 +- .../integrations/f5-ingresslink.md | 94 ++ .../installation/integrations/opentracing.md | 113 +++ .../content/installation/nic-images/_index.md | 5 + .../pulling-ingress-controller-image.md | 158 +++ .../nic-images/using-aws-marketplace-image.md | 89 ++ .../using-gcp-marketplace-package.md | 115 +++ .../using-the-jwt-token-docker-secret.md | 198 ++++ .../pulling-ingress-controller-image.md | 109 --- .../running-multiple-ingress-controllers.md | 49 +- .../using-aws-marketplace-image.md | 86 -- .../using-gcp-marketplace-package.md | 139 --- .../using-the-jwt-token-docker-secret.md | 189 ---- .../how-nginx-ingress-controller-works.md | 327 ------- docs/content/intro/nginx-plus.md | 35 - docs/content/intro/overview.md | 41 - .../logging-and-monitoring/prometheus.md | 22 +- docs/content/{intro => overview}/_index.md | 4 +- docs/content/overview/about.md | 16 + .../{intro => overview}/control-loop.png | Bin .../controller-comparison.md} | 12 +- .../{intro => overview}/controller-sync.png | Bin docs/content/overview/design.md | 312 ++++++ .../{intro => overview}/ic-high-level.png | Bin docs/content/{intro => overview}/ic-pod.png | Bin .../ic-process-components.png | Bin .../{intro => overview}/ic-process.png | Bin docs/content/overview/nginx-plus.md | 33 + docs/content/releases.md | 897 +++++++++--------- docs/content/technical-specifications.md | 108 +-- docs/content/third-party-modules/_index.md | 8 - .../third-party-modules/opentracing.md | 103 -- .../troubleshooting/troubleshoot-common.md | 2 +- docs/content/tutorials/custom-listen-ports.md | 2 +- .../tutorials/oidc-custom-configuration.md | 2 +- ...rtual-server-with-custom-listener-ports.md | 2 +- docs/content/usage-reporting.md | 2 +- docs/layouts/shortcodes/call-out.html | 3 + docs/layouts/shortcodes/custom-styles.html | 41 + 81 files changed, 3749 insertions(+), 3021 deletions(-) delete mode 100644 docs/content/app-protect-dos/installation.md delete mode 100644 docs/content/f5-ingresslink.md create mode 100644 docs/content/glossary.md create mode 100644 docs/content/includes/index.md create mode 100644 docs/content/includes/installation/create-common-resources.md create mode 100644 docs/content/includes/installation/manifests/daemonset.md create mode 100644 docs/content/includes/installation/manifests/deployment.md create mode 100644 docs/content/includes/installation/manifests/verify-pods-are-running.md create mode 100644 docs/content/includes/rbac/set-up-rbac.md delete mode 100644 docs/content/installation/building-ingress-controller-image.md create mode 100644 docs/content/installation/building-nginx-ingress-controller.md create mode 100644 docs/content/installation/ingress-nginx.md delete mode 100644 docs/content/installation/installation-with-helm.md delete mode 100644 docs/content/installation/installation-with-manifests.md delete mode 100644 docs/content/installation/installation-with-operator.md create mode 100644 docs/content/installation/installing-nic/_index.md create mode 100644 docs/content/installation/installing-nic/installation-with-helm.md create mode 100644 docs/content/installation/installing-nic/installation-with-manifests.md create mode 100644 docs/content/installation/installing-nic/installation-with-operator.md create mode 100644 docs/content/installation/integrations/_index.md rename docs/content/{ => installation/integrations}/app-protect-dos/_index.md (57%) rename docs/content/{ => installation/integrations}/app-protect-dos/configuration.md (86%) rename docs/content/{ => installation/integrations}/app-protect-dos/dos-protected.md (97%) create mode 100644 docs/content/installation/integrations/app-protect-dos/installation.md rename docs/content/{ => installation/integrations}/app-protect-waf/_index.md (80%) rename docs/content/{ => installation/integrations}/app-protect-waf/configuration.md (91%) rename docs/content/{ => installation/integrations}/app-protect-waf/installation.md (65%) create mode 100644 docs/content/installation/integrations/f5-ingresslink.md create mode 100644 docs/content/installation/integrations/opentracing.md create mode 100644 docs/content/installation/nic-images/_index.md create mode 100644 docs/content/installation/nic-images/pulling-ingress-controller-image.md create mode 100644 docs/content/installation/nic-images/using-aws-marketplace-image.md create mode 100644 docs/content/installation/nic-images/using-gcp-marketplace-package.md create mode 100644 docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md delete mode 100644 docs/content/installation/pulling-ingress-controller-image.md delete mode 100644 docs/content/installation/using-aws-marketplace-image.md delete mode 100644 docs/content/installation/using-gcp-marketplace-package.md delete mode 100644 docs/content/installation/using-the-jwt-token-docker-secret.md delete mode 100644 docs/content/intro/how-nginx-ingress-controller-works.md delete mode 100644 docs/content/intro/nginx-plus.md delete mode 100644 docs/content/intro/overview.md rename docs/content/{intro => overview}/_index.md (72%) create mode 100644 docs/content/overview/about.md rename docs/content/{intro => overview}/control-loop.png (100%) rename docs/content/{intro/nginx-ingress-controllers.md => overview/controller-comparison.md} (94%) rename docs/content/{intro => overview}/controller-sync.png (100%) create mode 100644 docs/content/overview/design.md rename docs/content/{intro => overview}/ic-high-level.png (100%) rename docs/content/{intro => overview}/ic-pod.png (100%) rename docs/content/{intro => overview}/ic-process-components.png (100%) rename docs/content/{intro => overview}/ic-process.png (100%) create mode 100644 docs/content/overview/nginx-plus.md delete mode 100644 docs/content/third-party-modules/_index.md delete mode 100644 docs/content/third-party-modules/opentracing.md create mode 100644 docs/layouts/shortcodes/call-out.html create mode 100644 docs/layouts/shortcodes/custom-styles.html diff --git a/README.md b/README.md index 59a075580aa..49b64855878 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ In the case of NGINX, the Ingress Controller is deployed in a pod along with the > page](https://github.com/nginxinc/kubernetes-ingress/releases) of the GitHub repository. 1. Install the NGINX Ingress Controller using the [Helm - chart](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-helm/) or the Kubernetes - [manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/). + chart](https://docs.nginx.com/nginx-ingress-controller/installation/installing-nic/installation-with-helm/) or the Kubernetes + [manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installing-nic/installation-with-manifests/). 1. Configure load balancing for a simple web application: - Use the Ingress resource. See the [Cafe example](https://github.com/nginxinc/kubernetes-ingress/tree/main/examples/ingress-resources/complete-example). @@ -98,7 +98,7 @@ In the case of NGINX, the Ingress Controller is deployed in a pod along with the We publish NGINX Ingress Controller releases on GitHub. See our [releases page](https://github.com/nginxinc/kubernetes-ingress/releases). -The latest stable release is [3.3.2](https://github.com/nginxinc/kubernetes-ingress/releases/tag/v3.3.2). For production +The latest stable release is [3.3.1](https://github.com/nginxinc/kubernetes-ingress/releases/tag/v3.3.1). For production use, we recommend that you choose the latest stable release. The edge version is useful for experimenting with new features that are not yet published in a stable release. To use @@ -118,7 +118,7 @@ your links to the correct versions: | Version | Description | Image for NGINX | Image for NGINX Plus | Installation Manifests and Helm Chart | Documentation and Examples | | ------- | ----------- | --------------- | -------------------- | ---------------------------------------| -------------------------- | -| Latest stable release | For production use | Use the 3.3.2 images from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress) or [build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | Use the 3.3.2 images from the [F5 Container Registry](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or [Build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | [Manifests](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/deployments). [Helm chart](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/deployments/helm-chart). | [Documentation](https://docs.nginx.com/nginx-ingress-controller/). [Examples](https://docs.nginx.com/nginx-ingress-controller/configuration/configuration-examples/). | +| Latest stable release | For production use | Use the 3.3.1 images from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress) or [build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | Use the 3.3.1 images from the [F5 Container Registry](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or [Build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | [Manifests](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.1/deployments). [Helm chart](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.1/deployments/helm-chart). | [Documentation](https://docs.nginx.com/nginx-ingress-controller/). [Examples](https://docs.nginx.com/nginx-ingress-controller/configuration/configuration-examples/). | | Edge/Nightly | For testing and experimenting | Use the edge or nightly images from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress) or [build your own image](https://github.com/nginxinc/kubernetes-ingress/tree/main/docs/content/installation/building-ingress-controller-image.md). | [Build your own image](https://github.com/nginxinc/kubernetes-ingress/tree/main/docs/content/installation/building-ingress-controller-image.md). | [Manifests](https://github.com/nginxinc/kubernetes-ingress/tree/main/deployments). [Helm chart](https://github.com/nginxinc/kubernetes-ingress/tree/main/charts/nginx-ingress). | [Documentation](https://github.com/nginxinc/kubernetes-ingress/tree/main/docs/content). [Examples](https://github.com/nginxinc/kubernetes-ingress/tree/main/examples). | ## SBOM (Software Bill of Materials) diff --git a/docs/content/app-protect-dos/installation.md b/docs/content/app-protect-dos/installation.md deleted file mode 100644 index 2b77b0dd349..00000000000 --- a/docs/content/app-protect-dos/installation.md +++ /dev/null @@ -1,85 +0,0 @@ ---- -title: Installation with NGINX App Protect DoS -description: "This document provides an overview of the steps required to use NGINX App Protect DoS with your NGINX Ingress Controller deployment." -weight: 1800 -doctypes: [""] -toc: true -docs: "DOCS-583" ---- - -> **Note**: The F5 NGINX Kubernetes Ingress Controller integration with F5 NGINX App Protect DoS requires the use of F5 NGINX Plus. - -This document provides an overview of the steps required to use NGINX App Protect DoS with your NGINX Ingress Controller deployment. You can visit the linked documents to find additional information and instructions. - -## Prerequisites - -1. Make sure you have access to the Ingress Controller image: - - For NGINX Plus Ingress Controller, see [here](/nginx-ingress-controller/installation/pulling-ingress-controller-image) for details on how to pull the image from the F5 Docker registry. - - To pull from the F5 Container registry in your Kubernetes cluster, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here](/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret). - - It is also possible to build your own image and push it to your private Docker registry by following the instructions from [here](/nginx-ingress-controller/installation/building-ingress-controller-image). -2. Clone the Ingress Controller repo: - - ``` - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 - cd kubernetes-ingress/deployments - ``` - -## Install the App Protect DoS Arbitrator - -### Helm Chart - -The App Protect DoS Arbitrator can be installed using the [NGINX App Protect DoS Helm Chart](https://github.com/nginxinc/nap-dos-arbitrator-helm-chart). -If you have the NGINX Helm Repository already added, you can install the App Protect DoS Arbitrator by running the following command: - -```console -helm install my-release-dos nginx-stable/nginx-appprotect-dos-arbitrator -``` - -### YAML Manifests - -Alternatively, you can install the App Protect DoS Arbitrator using the YAML manifests provided in the Ingress Controller repo. - -- Create the namespace and service account - -```console - kubectl apply -f common/ns-and-sa.yaml -``` - -- Deploy the app protect dos arbitrator - - ```console - kubectl apply -f deployment/appprotect-dos-arb.yaml - kubectl apply -f service/appprotect-dos-arb-svc.yaml - ``` - -## Build the Docker Image - -Take the steps below to create the Docker image that you'll use to deploy NGINX Ingress Controller with App Protect DoS in Kubernetes. - -- [Build the NGINX Ingress Controller image](/nginx-ingress-controller/installation/building-ingress-controller-image). - - When running the `make` command to build the image, be sure to use the `debian-image-dos-plus` target. For example: - - ```console - make debian-image-dos-plus PREFIX=/nginx-plus-ingress - ``` - - Alternatively, if you want to run on an [OpenShift](https://www.openshift.com/) cluster, use the `ubi-image-dos-plus` target. - - If you want to include the App Protect WAF module in the image, you can use the `debian-image-nap-dos-plus` target or the `ubi-image-nap-dos-plus` target for OpenShift. - -- [Push the image to your local Docker registry](/nginx-ingress-controller/installation/building-ingress-controller-image/#building-the-image-and-pushing-it-to-the-private-registry). - -## Install the Ingress Controller - -Take the steps below to set up and deploy the NGINX Ingress Controller and App Protect DoS module in your Kubernetes cluster. - -1. [Configure role-based access control (RBAC)](/nginx-ingress-controller/installation/installation-with-manifests/#1-configure-rbac). - - > **Important**: You must have an admin role to configure RBAC in your Kubernetes cluster. - -2. [Create the common Kubernetes resources](/nginx-ingress-controller/installation/installation-with-manifests/#2-create-common-resources). -3. Enable the App Protect Dos module by adding the `enable-app-protect-dos` [cli argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#cmdoption-enable-app-protect-dos) to your Deployment or DaemonSet file. -4. [Deploy the Ingress Controller](/nginx-ingress-controller/installation/installation-with-manifests/#3-deploy-the-ingress-controller). - -For more information, see the [Configuration guide](/nginx-ingress-controller/app-protect-dos/configuration),the [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-dos). diff --git a/docs/content/configuration/configuration-examples.md b/docs/content/configuration/configuration-examples.md index 94af4ce2396..181461df68c 100644 --- a/docs/content/configuration/configuration-examples.md +++ b/docs/content/configuration/configuration-examples.md @@ -11,5 +11,5 @@ docs: "DOCS-584" Our [GitHub repo](https://github.com/nginxinc/kubernetes-ingress) includes a number of configuration examples: -- [*Examples of Custom Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. -- [*Examples of Ingress Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. +- [*Examples of Custom Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. +- [*Examples of Ingress Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index e5a371b7034..fc718ecfc0a 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -7,14 +7,22 @@ toc: true docs: "DOCS-585" --- +{{< custom-styles >}} -The Ingress Controller supports several command-line arguments. Setting the arguments depends on how you install the Ingress Controller: + -- If you're using *Kubernetes manifests* (Deployment or DaemonSet) to install the Ingress Controller, to set the command-line arguments, modify those manifests accordingly. See the [Installation with Manifests](/nginx-ingress-controller/installation/installation-with-manifests) doc. -- If you're using *Helm* to install the Ingress Controller, modify the parameters of the Helm chart that correspond to the command-line arguments. See the [Installation with Helm](/nginx-ingress-controller/installation/installation-with-helm) doc. +NGINX Ingress Controller supports several command-line arguments. Setting the arguments depends on how you install NGINX Ingress Controller: + +- If you're using *Kubernetes manifests* (Deployment or DaemonSet) to install NGINX Ingress Controller, to set the command-line arguments, modify those manifests accordingly. See the [Installation with Manifests]({{}}) documentation. +- If you're using *Helm* to install NGINX Ingress Controller, modify the parameters of the Helm chart that correspond to the command-line arguments. See the [Installation with Helm]({{}}) documentation. Below we describe the available command-line arguments: -  + ### -enable-snippets @@ -22,7 +30,7 @@ Below we describe the available command-line arguments: Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. Default `false`. -  + ### -default-server-tls-secret `` @@ -30,11 +38,11 @@ Default `false`. Secret with a TLS certificate and key for TLS termination of the default server. - If not set, certificate and key in the file `/etc/nginx/secrets/default` are used. -- If `/etc/nginx/secrets/default` doesn't exist, the Ingress Controller will configure NGINX to reject TLS connections to the default server. -- If a secret is set, but the Ingress Controller is not able to fetch it from Kubernetes API, or it is not set and the Ingress Controller fails to read the file "/etc/nginx/secrets/default", the Ingress Controller will fail to start. +- If `/etc/nginx/secrets/default` doesn't exist, NGINX Ingress Controller will configure NGINX to reject TLS connections to the default server. +- If a secret is set, but NGINX Ingress Controller is not able to fetch it from Kubernetes API, or it is not set and NGINX Ingress Controller fails to read the file "/etc/nginx/secrets/default", NGINX Ingress Controller will fail to start. Format: `/` -  + ### -wildcard-tls-secret `` @@ -43,10 +51,10 @@ A Secret with a TLS certificate and key for TLS termination of every Ingress/Vir - If the argument is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection. -- If the argument is set, but the Ingress Controller is not able to fetch the Secret from Kubernetes API, the Ingress Controller will fail to start. +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. Format: `/` -  + ### -enable-custom-resources @@ -54,7 +62,15 @@ Format: `/` Enables custom resources. Default `true`. -  + + + +### -enable-preview-policies + +Enables preview policies. This flag is deprecated. To enable OIDC Policies please use [-enable-oidc](#cmdoption-enable-oidc) instead. + +Default `false`. + ### -enable-oidc @@ -62,7 +78,7 @@ Default `true`. Enables OIDC policies. Default `false`. -  + ### -inlcude-year @@ -79,7 +95,7 @@ Enables Leader election to avoid multiple replicas of the controller reporting t Default `true`. See [-report-ingress-status](#cmdoption-report-ingress-status) flag. -  + ### -enable-tls-passthrough @@ -87,7 +103,7 @@ See [-report-ingress-status](#cmdoption-report-ingress-status) flag. Enable TLS Passthrough on port 443. Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). -  + ### -tls-passthrough-port `` @@ -96,7 +112,7 @@ Set the port for TLS Passthrough. Format: `[1024 - 65535]` (default `443`) Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). -  + ### -enable-cert-manager @@ -104,7 +120,7 @@ Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). Enable x509 automated certificate management for VirtualServer resources using cert-manager (cert-manager.io). Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). -  + ### -enable-external-dns @@ -116,42 +132,42 @@ Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). ### -external-service `` -Specifies the name of the service with the type LoadBalancer through which the Ingress Controller pods are exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. +Specifies the name of the service with the type LoadBalancer through which the NGINX Ingress Controller pods are exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. For Ingress resources only: Requires [-report-ingress-status](#cmdoption-report-ingress-status). -  + ### -ingresslink `` -Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. +Specifies the name of the IngressLink resource, which exposes the NGINX Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. For Ingress resources only: Requires [-report-ingress-status](#cmdoption-report-ingress-status). -  + ### -global-configuration `` -A GlobalConfiguration resource for global configuration of the Ingress Controller. +A GlobalConfiguration resource for global configuration of NGINX Ingress Controller. Format: `/` Requires [-enable-custom-resources](#cmdoption-enable-custom-resources). -  + ### -health-status Adds a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. -Useful for external health-checking of the Ingress Controller. -  +Useful for external health-checking of NGINX Ingress Controller. + ### -health-status-uri `` Sets the URI of health status location in the default server. Requires [-health-status](#cmdoption-health-status). (default `/nginx-health`) -  + ### -ingress-class `` @@ -160,13 +176,13 @@ The `-ingress-class` argument refers to the name of the resource `kind: IngressC NGINX Ingress Controller will process Ingress resources that belong to its class, that is, those that have the `ingressClassName` field equal to the value of `-ingress-class` and skip the ones without it. It will also process all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the `ingressClassName` field. Default `nginx`. -  + ### -ingress-template-path `` Path to the ingress NGINX configuration template for an ingress resource. Default for NGINX is `nginx.ingress.tmpl`; default for NGINX Plus is `nginx-plus.ingress.tmpl`. -  + ### -leader-election-lock-name `` @@ -174,13 +190,13 @@ Path to the ingress NGINX configuration template for an ingress resource. Defaul Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. Requires [-enable-leader-election](#cmdoption-enable-leader-election). -  + ### -log_backtrace_at `` When logging hits line `file:N`, emit a stack trace. -  + ### -main-template-path `` @@ -189,35 +205,35 @@ Path to the main NGINX configuration template. - Default for NGINX is `nginx.ingress.tmpl`. - Default for NGINX Plus is `nginx-plus.ingress.tmpl`. -  + ### -nginx-configmaps `` -A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set, but the Ingress Controller is not able to fetch it from Kubernetes API, the Ingress Controller will fail to start. +A ConfigMap resource for customizing NGINX configuration. If a ConfigMap is set, but NGINX Ingress Controller is not able to fetch it from Kubernetes API, NGINX Ingress Controller will fail to start. Format: `/` -  + ### -nginx-debug Enable debugging for NGINX. Uses the nginx-debug binary. Requires 'error-log-level: debug' in the ConfigMap. -  + ### -nginx-plus Enable support for NGINX Plus. -  + ### -nginx-reload-timeout `` -Timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. +Timeout in milliseconds which NGINX Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. Default is 60000. -  + ### -nginx-status @@ -225,7 +241,7 @@ Default is 60000. Enable the NGINX stub_status, or the NGINX Plus API. Default `true`. -  + ### -nginx-status-allow-cidrs `` @@ -233,7 +249,7 @@ Default `true`. Add IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas. (default `127.0.0.1,::1`) -  + ### -nginx-status-port `` @@ -241,15 +257,15 @@ Separate multiple IP/CIDR by commas. (default `127.0.0.1,::1`) Set the port where the NGINX stub_status or the NGINX Plus API is exposed. Format: `[1024 - 65535]` (default `8080`) -  + ### -proxy `` Use a proxy server to connect to Kubernetes API started by "kubectl proxy" command. **For testing purposes only**. -The Ingress Controller does not start NGINX and does not write any generated NGINX configuration files to disk. -  +NGINX Ingress Controller does not start NGINX and does not write any generated NGINX configuration files to disk. + ### -report-ingress-status @@ -257,7 +273,7 @@ The Ingress Controller does not start NGINX and does not write any generated NGI Updates the address field in the status of Ingress resources. Requires the [-external-service](#cmdoption-external-service) or [-ingresslink](#cmdoption-ingresslink) flag, or the `external-status-address` key in the ConfigMap. -  + ### -transportserver-template-path `` @@ -267,19 +283,19 @@ Path to the TransportServer NGINX configuration template for a TransportServer r - Default for NGINX is `nginx.transportserver.tmpl`. - Default for NGINX Plus is `nginx-plus.transportserver.tmpl`. -  + ### -v `` Log level for V logs. -  + ### -version Print the version, git-commit hash and build date and exit. -  + ### -virtualserver-template-path `` @@ -289,37 +305,37 @@ Path to the VirtualServer NGINX configuration template for a VirtualServer resou - Default for NGINX is `nginx.ingress.tmpl`. - Default for NGINX Plus is `nginx-plus.ingress.tmpl`. -  + ### -vmodule `` A comma-separated list of pattern=N settings for file-filtered logging. -  + ### -watch-namespace `` -Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace-label". -  +Comma separated list of namespaces NGINX Ingress Controller should watch for resources. By default NGINX Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace-label". + ### -watch-namespace-label `` -Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace". -  +Configures NGINX Ingress Controller to watch only those namespaces with label foo=bar. By default NGINX Ingress Controller watches all namespaces. Mutually exclusive with "watch-namespace". + ### -watch-secret-namespace `` -Comma separated list of namespaces the Ingress Controller should watch for secrets. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See "watch-namespace" and "watch-namespace-label". -  +Comma separated list of namespaces NGINX Ingress Controller should watch for secrets. If this arg is not configured, NGINX Ingress Controller watches the same namespaces for all resources. See "watch-namespace" and "watch-namespace-label". + ### -enable-prometheus-metrics Enables exposing NGINX or NGINX Plus metrics in the Prometheus format. -  + ### -prometheus-metrics-listen-port `` @@ -327,7 +343,7 @@ Enables exposing NGINX or NGINX Plus metrics in the Prometheus format. Sets the port where the Prometheus metrics are exposed. Format: `[1024 - 65535]` (default `9113`) -  + ### -prometheus-tls-secret `` @@ -335,14 +351,14 @@ Format: `[1024 - 65535]` (default `9113`) A Secret with a TLS certificate and key for TLS termination of the Prometheus metrics endpoint. - If the argument is not set, the Prometheus endpoint will not use a TLS connection. -- If the argument is set, but the Ingress Controller is not able to fetch the Secret from Kubernetes API, the Ingress Controller will fail to start. -  +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. + ### -enable-service-insight Exposes the Service Insight endpoint for Ingress Controller. -  + ### -service-insight-listen-port `` @@ -350,7 +366,7 @@ Exposes the Service Insight endpoint for Ingress Controller. Sets the port where the Service Insight is exposed. Format: `[1024 - 65535]` (default `9114`) -  + ### -service-insight-tls-secret `` @@ -358,19 +374,19 @@ Format: `[1024 - 65535]` (default `9114`) A Secret with a TLS certificate and key for TLS termination of the Service Insight endpoint. - If the argument is not set, the Service Insight endpoint will not use a TLS connection. -- If the argument is set, but the Ingress Controller is not able to fetch the Secret from Kubernetes API, the Ingress Controller will fail to start. +- If the argument is set, but NGINX Ingress Controller is not able to fetch the Secret from Kubernetes API, NGINX Ingress Controller will fail to start. Format: `/` -  + ### -spire-agent-address `` Specifies the address of a running Spire agent. **For use with NGINX Service Mesh only**. -- If the argument is set, but the Ingress Controller is unable to connect to the Spire Agent, the Ingress Controller will fail to start. +- If the argument is set, but NGINX Ingress Controller is unable to connect to the Spire Agent, NGINX Ingress Controller will fail to start. + -  ### -enable-internal-routes @@ -379,16 +395,16 @@ Enable support for internal routes with NGINX Service Mesh. **For use with NGINX Requires [-spire-agent-address](#cmdoption-spire-agent-address). -- If the argument is set, but `spire-agent-address` is not provided, the Ingress Controller will fail to start. +- If the argument is set, but `spire-agent-address` is not provided, NGINX Ingress Controller will fail to start. + -  ### -enable-latency-metrics Enable collection of latency metrics for upstreams. Requires [-enable-prometheus-metrics](#cmdoption-enable-prometheus-metrics). -  + ### -enable-app-protect @@ -397,9 +413,9 @@ Enables support for App Protect. Requires [-nginx-plus](#cmdoption-nginx-plus). -- If the argument is set, but `nginx-plus` is set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will fail to start. + -  ### -app-protect-log-level `` @@ -408,9 +424,9 @@ Sets log level for App Protect. Allowed values: fatal, error, warn, info, debug, Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect](#cmdoption-enable-app-protect). -- If the argument is set, but `nginx-plus` and `enable-app-protect` are set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` and `enable-app-protect` are set to false, NGINX Ingress Controller will fail to start. + -  ### -enable-app-protect-dos @@ -419,9 +435,9 @@ Enables support for App Protect DoS. Requires [-nginx-plus](#cmdoption-nginx-plus). -- If the argument is set, but `nginx-plus` is set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` is set to false, NGINX Ingress Controller will fail to start. + -  ### -app-protect-dos-debug @@ -430,9 +446,9 @@ Enable debugging for App Protect DoS. Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). -- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + -  ### -app-protect-dos-max-daemons @@ -443,9 +459,9 @@ Default `1`. Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). -- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + -  ### -app-protect-dos-max-workers @@ -456,9 +472,9 @@ Default `Number of CPU cores in the machine`. Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). -- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. + -  ### -app-protect-dos-memory @@ -469,9 +485,8 @@ Default `50% of free RAM in the container or 80MB, the smaller`. Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmdoption-enable-app-protect-dos). -- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, the Ingress Controller will fail to start. +- If the argument is set, but `nginx-plus` and `enable-app-protect-dos` are set to false, NGINX Ingress Controller will fail to start. -  ### -ready-status @@ -479,7 +494,7 @@ Requires [-nginx-plus](#cmdoption-nginx-plus) and [-enable-app-protect-dos](#cmd Enables the readiness endpoint `/nginx-ready`. The endpoint returns a success code when NGINX has loaded all the config after the startup. Default `true`. -  + ### -ready-status-port @@ -487,28 +502,12 @@ Default `true`. The HTTP port for the readiness endpoint. Format: `[1024 - 65535]` (default `8081`) -  + ### -disable-ipv6 Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. Default `false`. -  - - -### -default-http-listener-port - -Sets the port for the HTTP `default_server` listener. -Default `80`. -  - - -### -default-https-listener-port - -Sets the port for the HTTPS `default_server` listener. - -Default `443`. -  - + diff --git a/docs/content/configuration/global-configuration/configmap-resource.md b/docs/content/configuration/global-configuration/configmap-resource.md index f8785d2d401..ca3ced6b44b 100644 --- a/docs/content/configuration/global-configuration/configmap-resource.md +++ b/docs/content/configuration/global-configuration/configmap-resource.md @@ -91,10 +91,10 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``server-names-hash-max-size`` | Sets the value of the [server_names_hash_max_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) directive. | ``1024`` | | |``map-hash-bucket-size`` | Sets the value of the [map_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size) directive.| ``256`` | | |``map-hash-max-size`` | Sets the value of the [map_hash_max_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_max_size) directive. | ``2048`` | | -|``resolver-addresses`` | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, ``kube-dns.kube-system.svc.cluster.local`` ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | -|``resolver-ipv6`` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | ``True`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | -|``resolver-valid`` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | -|``resolver-timeout`` | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | ``30s`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | +|``resolver-addresses`` | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, ``kube-dns.kube-system.svc.cluster.local`` ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | +|``resolver-ipv6`` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | ``True`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | +|``resolver-valid`` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | +|``resolver-timeout`` | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | ``30s`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | |``keepalive-timeout`` | Sets the value of the [keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) directive. | ``65s`` | | |``keepalive-requests`` | Sets the value of the [keepalive_requests](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) directive. | ``100`` | | |``variables-hash-bucket-size`` | Sets the value of the [variables_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_bucket_size) directive. | ``256`` | | @@ -109,9 +109,9 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``error-log-level`` | Sets the global [error log level](https://nginx.org/en/docs/ngx_core_module.html#error_log) for NGINX. | ``notice`` | | |``access-log-off`` | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). | ``False`` | | |``default-server-access-log-off`` | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) for the default server. If access log is disabled globally (``access-log-off: "True"``), then the default server access log is always disabled. | ``False`` | | -|``log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/shared-examples/custom-log-format). | +|``log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/shared-examples/custom-log-format). | |``log-format-escaping`` | Sets the characters escaping for the variables of the log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | -|``stream-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx.tmpl). | | +|``stream-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx.tmpl). | | |``stream-log-format-escaping`` | Sets the characters escaping for the variables of the stream log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | {{% /table %}} @@ -147,7 +147,7 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |ConfigMap Key | Description | Default | Example | | ---| ---| ---| --- | |``http2`` | Enables HTTP/2 in servers with SSL enabled. | ``False`` | | -|``proxy-protocol`` | Enables PROXY Protocol for incoming connections. | ``False`` | [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/shared-examples/proxy-protocol). | +|``proxy-protocol`` | Enables PROXY Protocol for incoming connections. | ``False`` | [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/shared-examples/proxy-protocol). | {{% /table %}} ### Backend Services (Upstreams) @@ -171,13 +171,13 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``http-snippets`` | Sets a custom snippet in http context. | N/A | | |``location-snippets`` | Sets a custom snippet in location context. | N/A | | |``server-snippets`` | Sets a custom snippet in server context. | N/A | | -|``stream-snippets`` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/tcp-udp). | +|``stream-snippets`` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/tcp-udp). | |``main-template`` | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | |``ingress-template`` | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | |``virtualserver-template`` | Sets the NGINX configuration template for an VirtualServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | {{% /table %}} -### Modules +### Modules {#modules} {{% table %}} |ConfigMap Key | Description | Default | Example | diff --git a/docs/content/configuration/global-configuration/custom-templates.md b/docs/content/configuration/global-configuration/custom-templates.md index a3d1c51c8ec..35b70302a89 100644 --- a/docs/content/configuration/global-configuration/custom-templates.md +++ b/docs/content/configuration/global-configuration/custom-templates.md @@ -9,4 +9,4 @@ docs: "DOCS-587" --- -The Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/shared-examples/custom-templates). +The Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/shared-examples/custom-templates). diff --git a/docs/content/configuration/global-configuration/globalconfiguration-resource.md b/docs/content/configuration/global-configuration/globalconfiguration-resource.md index bfa29d6077a..e870fc06151 100644 --- a/docs/content/configuration/global-configuration/globalconfiguration-resource.md +++ b/docs/content/configuration/global-configuration/globalconfiguration-resource.md @@ -11,8 +11,7 @@ docs: "DOCS-588" The GlobalConfiguration resource allows you to define the global configuration parameters of the Ingress Controller. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -The resource supports configuring listeners for TCP and UDP load balancing. Listeners are required by [TransportServer resources](/nginx-ingress-controller/configuration/transportserver-resource) and -can be used to configure custom listerners for VirtualServers as specified [here](/nginx-ingress-controller/tutorials/virtual-server-with-custom-listener-ports). +The resource supports configuring listeners for TCP and UDP load balancing. Listeners are required by [TransportServer resources](/nginx-ingress-controller/configuration/transportserver-resource). ## Prerequisites @@ -23,7 +22,7 @@ When [installing](/nginx-ingress-controller/installation/installation-with-manif The GlobalConfiguration resource defines the global configuration parameters of the Ingress Controller. Below is an example: ```yaml -apiVersion: k8s.nginx.org/v1 +apiVersion: k8s.nginx.org/v1alpha1 kind: GlobalConfiguration metadata: name: nginx-configuration @@ -36,13 +35,6 @@ spec: - name: dns-tcp port: 5353 protocol: TCP - - name: http-8083 - port: 8083 - protocol: HTTP - - name: https-8443 - port: 8443 - protocol: HTTP - ssl: true ``` {{% table %}} @@ -53,15 +45,12 @@ spec: ### Listener -The `listeners:` key defines a listener (a combination of a protocol and a port) that NGINX will use to accept traffic for a [TransportServer](/nginx-ingress-controller/configuration/transportserver-resource) and a [VirtualServer](nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources): +The listener defines a listener (a combination of a protocol and a port) that NGINX will use to accept traffic for a [TransportServer](/nginx-ingress-controller/configuration/transportserver-resource): ```yaml -- name: dns-tcp - port: 5353 - protocol: TCP -- name: http-8083 - port: 8083 - protocol: HTTP +name: dns-tcp +port: 5353 +protocol: TCP ``` {{% table %}} @@ -111,7 +100,7 @@ If you try to create (or update) a resource that violates the structural schema ``` $ kubectl apply -f global-configuration.yaml - error: error validating "global-configuration.yaml": error validating data: ValidationError(GlobalConfiguration.spec.listeners[0].port): invalid type for org.nginx.k8s.v1.GlobalConfiguration.spec.listeners.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + error: error validating "global-configuration.yaml": error validating data: ValidationError(GlobalConfiguration.spec.listeners[0].port): invalid type for org.nginx.k8s.v1alpha1.GlobalConfiguration.spec.listeners.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false ``` - Example of Kubernetes API server validation: diff --git a/docs/content/configuration/handling-host-and-listener-collisions.md b/docs/content/configuration/handling-host-and-listener-collisions.md index 8d176169137..483544408f8 100644 --- a/docs/content/configuration/handling-host-and-listener-collisions.md +++ b/docs/content/configuration/handling-host-and-listener-collisions.md @@ -87,7 +87,7 @@ Similarly, if `cafe-ingress` was created first, it will win `cafe.example.com` a It is possible to merge configuration for multiple Ingress resources for the same host. One common use case for this approach is distributing resources across multiple namespaces. See the [Cross-namespace Configuration](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration/) doc for more information. -It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/cross-namespace-configuration) on GitHub. +It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/cross-namespace-configuration) on GitHub. It is *not* possible to merge configuration for multiple TransportServer resources. @@ -102,7 +102,7 @@ Consider the following two resources: - `tcp-1` TransportServer: ```yaml - apiVersion: k8s.nginx.org/v1 + apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: tcp-1 @@ -116,7 +116,7 @@ Consider the following two resources: - `tcp-2` TransportServer: ```yaml - apiVersion: k8s.nginx.org/v1 + apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: tcp-2 diff --git a/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md b/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md index e2309ed995c..ff71d698680 100644 --- a/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md +++ b/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md @@ -122,7 +122,7 @@ The table below summarizes the available annotations. | ---| ---| ---| ---| --- | |``nginx.org/proxy-hide-headers`` | ``proxy-hide-headers`` | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: ``"nginx.org/proxy-hide-headers": "header-a,header-b"`` | N/A | | |``nginx.org/proxy-pass-headers`` | ``proxy-pass-headers`` | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: ``"nginx.org/proxy-pass-headers": "header-a,header-b"`` | N/A | | -|``nginx.org/rewrites`` | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [Rewrites Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/rewrites). | +|``nginx.org/rewrites`` | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [Rewrites Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/rewrites). | {{% /table %}} ### Auth and SSL/TLS @@ -138,10 +138,10 @@ The table below summarizes the available annotations. |``nginx.org/hsts-behind-proxy`` | ``hsts-behind-proxy`` | Enables HSTS based on the value of the ``http_x_forwarded_proto`` request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of the Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the ``nginx.org/redirect-to-https`` annotation. | ``False`` | | |``nginx.org/basic-auth-secret`` | N/A | Specifies a Secret resource with a user list for HTTP Basic authentication. | N/A | | |``nginx.org/basic-auth-realm`` | N/A | Specifies a realm. | N/A | | -|``nginx.com/jwt-key`` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | -|``nginx.com/jwt-realm`` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | -|``nginx.com/jwt-token`` | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization`` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | -|``nginx.com/jwt-login-url`` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-key`` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | +|``nginx.com/jwt-realm`` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | +|``nginx.com/jwt-token`` | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization`` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | +|``nginx.com/jwt-login-url`` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | {{% /table %}} ### Listeners @@ -159,19 +159,19 @@ The table below summarizes the available annotations. |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | |``nginx.org/lb-method`` | ``lb-method`` | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``"round_robin"``. | ``"random two least_conn"`` | | -|``nginx.org/ssl-services`` | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [SSL Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/ssl-services). | -|``nginx.org/grpc-services`` | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2`` ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [GRPC Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/grpc-services). | -|``nginx.org/websocket-services`` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/websocket). | +|``nginx.org/ssl-services`` | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [SSL Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/ssl-services). | +|``nginx.org/grpc-services`` | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2`` ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [GRPC Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/grpc-services). | +|``nginx.org/websocket-services`` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/websocket). | |``nginx.org/max-fails`` | ``max-fails`` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the ``server`` directive. | ``1`` | | |``nginx.org/max-conns`` | N\A | Sets the value of the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the ``server`` directive. | ``0`` | | |``nginx.org/upstream-zone-size`` | ``upstream-zone-size`` | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | ``256K`` | | |``nginx.org/fail-timeout`` | ``fail-timeout`` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the ``server`` directive. | ``10s`` | | -|``nginx.com/sticky-cookie-services`` | N/A | Configures session persistence. | N/A | [Session Persistence](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/session-persistence). | +|``nginx.com/sticky-cookie-services`` | N/A | Configures session persistence. | N/A | [Session Persistence](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/session-persistence). | |``nginx.org/keepalive`` | ``keepalive`` | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that ``proxy_set_header Connection "";`` is added to the generated configuration when the value > 0. | ``0`` | | -|``nginx.com/health-checks`` | N/A | Enables active health checks. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | -|``nginx.com/health-checks-mandatory`` | N/A | Configures active health checks as mandatory. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | -|``nginx.com/health-checks-mandatory-queue`` | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | ``0`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | -|``nginx.com/slow-start`` | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | ``"0s"`` | | +|``nginx.com/health-checks`` | N/A | Enables active health checks. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | +|``nginx.com/health-checks-mandatory`` | N/A | Configures active health checks as mandatory. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | +|``nginx.com/health-checks-mandatory-queue`` | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | ``0`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | +|``nginx.com/slow-start`` | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | ``"0s"`` | | {{% /table %}} ### Snippets and Custom Templates @@ -183,18 +183,18 @@ The table below summarizes the available annotations. |``nginx.org/server-snippets`` | ``server-snippets`` | Sets a custom snippet in server context. | N/A | | {{% /table %}} -### App Protect +### App Protect {#app-protect} **Note**: The App Protect annotations only work if App Protect WAF module is [installed](/nginx-ingress-controller/app-protect/installation/). {{% table %}} |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | -|``appprotect.f5.com/app-protect-policy`` | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable`` is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-enable`` | N/A | Enable App Protect for the Ingress Resource. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log-enable`` | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log`` | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log-destination`` | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | ``syslog:server=localhost:514`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-policy`` | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable`` is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-enable`` | N/A | Enable App Protect for the Ingress Resource. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log-enable`` | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log`` | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log-destination`` | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | ``syslog:server=localhost:514`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | {{% /table %}} ### App Protect DoS @@ -204,5 +204,5 @@ The table below summarizes the available annotations. {{% table %}} |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | -|``appprotectdos.f5.com/app-protect-dos-resource`` | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource](/nginx-ingress-controller/app-protect-dos/dos-protected/). | N/A | [Example for App Protect DoS](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-dos). | +|``appprotectdos.f5.com/app-protect-dos-resource`` | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource](/nginx-ingress-controller/app-protect-dos/dos-protected/). | N/A | [Example for App Protect DoS](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-dos). | {{% /table %}} diff --git a/docs/content/configuration/ingress-resources/basic-configuration.md b/docs/content/configuration/ingress-resources/basic-configuration.md index 9bf818470bd..6aba7b31d3f 100644 --- a/docs/content/configuration/ingress-resources/basic-configuration.md +++ b/docs/content/configuration/ingress-resources/basic-configuration.md @@ -53,7 +53,7 @@ Here is a breakdown of what this Ingress resource definition means: - The rule with the path `/coffee` instructs NGINX to distribute the requests with the `/coffee` URI among the pods of the *coffee* service, which is deployed with the name `coffee‑svc` in the cluster. - Both rules instruct NGINX to distribute the requests to `port 80` of the corresponding service (the `servicePort` field). -> For complete instructions on deploying the Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/complete-example) in our GitHub repository. +> For complete instructions on deploying the Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/complete-example) in our GitHub repository. > To learn more about the Ingress resource, see the [Ingress resource documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) in the Kubernetes docs. @@ -113,7 +113,7 @@ The NGINX Ingress Controller imposes the following restrictions on Ingress resou - When defining an Ingress resource, the `host` field is required. - The `host` value needs to be unique among all Ingress and VirtualServer resources unless the Ingress resource is a [mergeable minion](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration/). See also [Handling Host and Listener Collisions](/nginx-ingress-controller/configuration/handling-host-and-listener-collisions). - The `path` field in `spec.rules[].http.paths[]` is required for `Exact` and `Prefix` `pathTypes`. -- The ImplementationSpecific `pathType` is treated as equivalent to `Prefix` `pathType`, with the exception that when this `pathType` is configured, the `path` field in `spec.rules[].http.paths[]` is not mandatory. `path` defaults to `/` if not set but the `pathType` is set to ImplementationSpecific. +- The ImplementationSpecific `pathType` is treated as equivilent to `Prefix` `pathType`, with the exception that when this `pathType` is configured, the `path` field in `spec.rules[].http.paths[]` is not mandatory. `path` defaults to `/` if not set but the `pathType` is set to ImplementationSpecific. ## Advanced Configuration diff --git a/docs/content/configuration/ingress-resources/cross-namespace-configuration.md b/docs/content/configuration/ingress-resources/cross-namespace-configuration.md index ca0fbf090aa..3edb02ad58f 100644 --- a/docs/content/configuration/ingress-resources/cross-namespace-configuration.md +++ b/docs/content/configuration/ingress-resources/cross-namespace-configuration.md @@ -9,6 +9,6 @@ docs: "DOCS-594" --- -You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. +You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. -As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. +As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. diff --git a/docs/content/configuration/ingress-resources/custom-annotations.md b/docs/content/configuration/ingress-resources/custom-annotations.md index bd927d3b084..683114c98c0 100644 --- a/docs/content/configuration/ingress-resources/custom-annotations.md +++ b/docs/content/configuration/ingress-resources/custom-annotations.md @@ -23,7 +23,7 @@ Custom annotations allow you to add an annotation for an NGINX feature that is n ## Usage -The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx-plus.ingress.tmpl). +The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx-plus.ingress.tmpl). To support custom annotations, the template has access to the information about the Ingress resource - its *name*, *namespace* and *annotations*. It is possible to check if a particular annotation present in the Ingress resource and conditionally insert NGINX configuration directives at multiple NGINX contexts - `http`, `server`, `location` or `upstream`. Additionally, you can get the value that is set to the annotation. @@ -107,15 +107,10 @@ If you'd like to use custom annotations with Mergeable Ingress resources, please Helper functions can be used in the Ingress template to parse the values of custom annotations. {{% table %}} -| Function | Input Arguments | Return Arguments | Description | +|Function | Input Arguments | Return Arguments | Description | | ---| ---| ---| --- | -| ``split`` | ``s, sep string`` | ``[]string`` | Splits the string ``s`` into a slice of strings separated by the ``sep``. | -| ``trim`` | ``s string`` | ``string`` | Trims the trailing and leading whitespace from the string ``s``. | -| ``contains`` | ``s, substr string`` | ``bool`` | Tests whether the string ``substr`` is a substring of the string ``s``. | -| ``hasPrefix`` | ``s, prefix string`` | ``bool`` | Tests whether the string ``prefix`` is a prefix of the string ``s``. | -| ``hasSuffix`` | ``s, suffix string`` | ``bool`` | Tests whether the string ``suffix`` is a suffix of the string ``s``. | -| ``toLower`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their lower case. | -| ``toUpper`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their upper case. | +|``split`` | ``s, sep string`` | ``[]string`` | Splits the string ``s`` into a slice of strings separated by the ``sep``. | +|``trim`` | ``s string`` | ``string`` | Trims the trailing and leading whitespace from the string ``s``. | {{% /table %}} Consider the following custom annotation `custom.nginx.org/allowed-ips`, which expects a comma-separated list of IP addresses: @@ -144,4 +139,4 @@ deny all; ## Example -See the [custom annotations example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/ingress-resources/custom-annotations). +See the [custom annotations example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/ingress-resources/custom-annotations). diff --git a/docs/content/configuration/policy-resource.md b/docs/content/configuration/policy-resource.md index a24a9a00d29..6fc4c512359 100644 --- a/docs/content/configuration/policy-resource.md +++ b/docs/content/configuration/policy-resource.md @@ -12,7 +12,7 @@ The Policy resource allows you to configure features like access control and rat The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/access-control). +This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/access-control). ## Prerequisites @@ -459,7 +459,7 @@ NGINX Plus will pass the ID of an authenticated user to the backend in the HTTP #### Prerequisites In order to use OIDC, you need to enable [zone synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync/). If you don't set up zone synchronization, NGINX Plus will fail to reload. -You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). +You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). > **Note**: The configuration in the example doesn't enable TLS and the synchronization between the replica happens in clear text. This could lead to the exposure of tokens. @@ -519,7 +519,7 @@ webapp-policy 27m For `kubectl get` and similar commands, you can also use the short name `pol` instead of `policy`. -### WAF +### WAF {#waf} > Note: This feature is only available in NGINX Plus with AppProtect. diff --git a/docs/content/configuration/security.md b/docs/content/configuration/security.md index 472369b4500..9097b580b70 100644 --- a/docs/content/configuration/security.md +++ b/docs/content/configuration/security.md @@ -26,7 +26,7 @@ We strongly recommend using the [RBAC configuration](https://github.com/nginxinc It is configured with the least amount of privilege required for the Ingress Controller to work. We strongly recommend inspecting the RBAC configuration (for [manifests installation](https://github.com/nginxinc/kubernetes-ingress/blob/main/deployments/rbac/rbac.yaml) -or for [helm](https://github.com/nginxinc/kubernetes-ingress/blob/main/charts/nginx-ingress/templates/rbac.yaml)) +or for [helm](https://github.com/nginxinc/kubernetes-ingress/blob/main/deployments/helm-chart/templates/rbac.yaml)) to understand what access the Ingress Controller service account has and to which resources. For example, by default the service account has access to all Secret resources in the cluster. diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index bfed15b4777..62ae490d043 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -10,7 +10,7 @@ docs: "DOCS-598" The TransportServer resource allows you to configure TCP, UDP, and TLS Passthrough load balancing. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the TransportServer resource. To see additional examples of using the resource for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources) folder in our GitHub repo. +This document is the reference documentation for the TransportServer resource. To see additional examples of using the resource for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources) folder in our GitHub repo. ## Prerequisites @@ -24,7 +24,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, - TCP load balancing: ```yaml - apiVersion: k8s.nginx.org/v1 + apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: dns-tcp @@ -45,7 +45,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, - UDP load balancing: ```yaml - apiVersion: k8s.nginx.org/v1 + apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: dns-udp @@ -67,7 +67,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, - TLS passthrough load balancing: ```yaml - apiVersion: k8s.nginx.org/v1 + apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: secure-app @@ -299,7 +299,7 @@ In the kubectl get and similar commands, you can also use the short name `ts` in Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure [access control](http://nginx.org/en/docs/stream/ngx_stream_access_module.html) in a TransportServer: ```yaml -apiVersion: k8s.nginx.org/v1 +apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: cafe @@ -317,7 +317,7 @@ spec: Snippets can also be specified for a stream. In the example below, we use snippets to [limit the number of connections](https://nginx.org/en/docs/stream/ngx_stream_limit_conn_module.html): ```yaml -apiVersion: k8s.nginx.org/v1 +apiVersion: k8s.nginx.org/v1alpha1 kind: TransportServer metadata: name: cafe @@ -365,7 +365,7 @@ If you try to create (or update) a resource that violates the structural schema ```console kubectl apply -f transport-server-passthrough.yaml - error: error validating "transport-server-passthrough.yaml": error validating data: ValidationError(TransportServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1.TransportServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + error: error validating "transport-server-passthrough.yaml": error validating data: ValidationError(TransportServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1alpha1.TransportServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false ``` - Example of Kubernetes API server validation: diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md index ad9eee5d76d..d9bb7c6ef91 100644 --- a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md +++ b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md @@ -12,7 +12,7 @@ docs: "DOCS-599" The VirtualServer and VirtualServerRoute resources, introduced in release 1.5, enable use cases not supported with the Ingress resource, such as traffic splitting and advanced content-based routing. The resources are implemented as [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the resources. To see additional examples of using the resources for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources) folder in our GitHub repo. +This document is the reference documentation for the resources. To see additional examples of using the resources for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources) folder in our GitHub repo. ## VirtualServer Specification @@ -127,7 +127,6 @@ cert-manager: |``duration`` | This field allows you to configure spec.duration field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | |``renew-before`` | this annotation allows you to configure spec.renewBefore field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | |``usages`` | This field allows you to configure spec.usages field for the Certificate to be generated. Pass a string with comma-separated values i.e. ``key agreement,digital signature, server auth``. An exhaustive list of supported key usages can be found in the [the cert-manager api documentation](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.KeyUsage). | ``string`` | No | -|``issue-temp-cert`` | When ``true``, ask cert-manager for a [temporary self-signed certificate](https://cert-manager.io/docs/usage/certificate/#temporary-certificates-while-issuing) pending the issuance of the Certificate. This allows HTTPS-only servers to use ACME HTTP01 challenges when the TLS secret does not exist yet. | ``boolean`` | No | {{% /table %}} ### VirtualServer.Listener @@ -347,7 +346,7 @@ tls: |Field | Description | Type | Required | | ---| ---| ---| --- | |``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | -|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | |``subselector`` | Selects the pods within the service using label keys and values. By default, all pods of the service are selected. Note: the specified labels are expected to be present in the pods when they are created. If the pod labels are updated, the Ingress Controller will not see that change until the number of the pods is changed. | ``map[string]string`` | No | |``use-cluster-ip`` | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method`` and ``next-upstream``) will have no effect, as the Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | ``boolean`` | No | |``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. The port must fall into the range ``1..65535``. | ``uint16`` | Yes | @@ -634,7 +633,7 @@ proxy: |``upstream`` | The name of the upstream which the requests will be proxied to. The upstream with that name must be defined in the resource. | ``string`` | Yes | |``requestHeaders`` | The request headers modifications. | [action.Proxy.RequestHeaders](#actionproxyrequestheaders) | No | |``responseHeaders`` | The response headers modifications. | [action.Proxy.ResponseHeaders](#actionproxyresponseheaders) | No | -|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/rewrites) example. | ``string`` | No | +|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/rewrites) example. | ``string`` | No | {{% /table %}} ### Action.Proxy.RequestHeaders diff --git a/docs/content/f5-ingresslink.md b/docs/content/f5-ingresslink.md deleted file mode 100644 index 2268ed3548b..00000000000 --- a/docs/content/f5-ingresslink.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -title: Using with F5 BIG-IP -description: | - Learn how to use NGINX Ingress Controller with F5 IngressLink to configure your F5 BIG-IP device. -weight: 2000 -doctypes: ["concept"] -toc: true -docs: "DOCS-600" ---- - - -F5 IngressLink is the integration between NGINX Ingress Controller and [F5 Container Ingress Services](https://clouddocs.f5.com/containers/v2/) (CIS) that configures an F5 BIG-IP device as a load balancer for NGINX Ingress Controller pods. - -## Configuration - -### 1. Install the Ingress Controller with the Integration Enabled - -This step depends on how you install the Ingress Controller: using [Manifests](/nginx-ingress-controller/installation/installation-with-manifests) or the [Helm chart](/nginx-ingress-controller/installation/installation-with-helm). - -#### Manifests Installation - -1. Create a service for the Ingress Controller pods for ports 80 and 443. For example: - - ```yaml - apiVersion: v1 - kind: Service - metadata: - name: nginx-ingress-ingresslink - namespace: nginx-ingress - labels: - app: ingresslink - spec: - ports: - - port: 80 - targetPort: 80 - protocol: TCP - name: http - - port: 443 - targetPort: 443 - protocol: TCP - name: https - selector: - app: nginx-ingress - ``` - - Note the label `app: ingresslink`. We will use it in the Step 2. -1. In the [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource), enable the PROXY protocol, which the BIG-IP system will use to pass the client IP and port information to NGINX. For the `set-real-ip-from` key, use the subnet of the IP, which the BIG-IP system uses to send traffic to NGINX: - - ```yaml - proxy-protocol: "True" - real-ip-header: "proxy_protocol" - set-real-ip-from: "0.0.0.0/0" - ``` - -1. Deploy the Ingress Controller with additional [command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments): - - ```yaml - args: - - -ingresslink=nginx-ingress - - -report-ingress-status - . . . - ``` - - where `ingresslink` references the name of the IngressLink resource from Step 2, and `report-ingress-status` enables [reporting Ingress statuses](/nginx-ingress-controller/configuration/global-configuration/reporting-resources-status#ingress-resources). - -#### Helm Installation - -Install a helm release with the following values that replicate the Manifest installation above: - -```yaml -controller: - config: - entries: - proxy-protocol: "True" - real-ip-header: "proxy_protocol" - set-real-ip-from: "0.0.0.0/0" - reportIngressStatus: - ingressLink: nginx-ingress - service: - type: ClusterIP - externalTrafficPolicy: Cluster - extraLabels: - app: ingresslink -``` - -We will use the values for the parameters `ingressLink` and `extraLabels` in Step 2. For the `set-real-ip-from` key, use the subnet of the IP, which the BIG-IP system uses to send traffic to NGINX. - -### 2. Configure CIS - -To enable the integration, the F5 CIS must be deployed in the cluster and configured to support the integration. Follow the instructions on the [CIS documentation portal](https://clouddocs.f5.com/containers/latest/userguide/ingresslink/#configuring-ingresslink). Ensure that: - -- The name of the IngressLink resource is the same as in Step 1 -- `nginx-ingress`. -- The selector in the IngressLink resource is the same as the Service labels configured in Step 1 -- `app: ingresslink`. -- The IngressLink must belong to the same namespace as the Ingress Controller pod -- `nginx-ingress` or the namespace used for installing the Helm chart. diff --git a/docs/content/glossary.md b/docs/content/glossary.md new file mode 100644 index 00000000000..c9ca266b266 --- /dev/null +++ b/docs/content/glossary.md @@ -0,0 +1,35 @@ +--- +title: Glossary +description: +weight: 10000 +menu: + docs: + parent: NGINX Ingress Controller +--- + +{{}} + + + +## Ingress {#ingress} + +_Ingress_ refers to an _Ingress Resource_, a Kubernetes API object which allows access to [Services](https://kubernetes.io/docs/concepts/services-networking/service/) within a cluster. They are managed by an [Ingress Controller]({{< relref "glossary.md#ingress-controller">}}). + +_Ingress_ resources enable the following functionality: + +- **Load balancing**, extended through the use of Services +- **Content-based routing**, using hosts and paths +- **TLS/SSL termination**, based on hostnames + +For additional information, please read the official [Kubernetes Ingress Documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/). + +## Ingress Controller {#ingress-controller} + +*Ingress Controllers* are applications within a Kubernetes cluster that enable [Ingress]({{< relref "glossary.md#ingress">}}) resources to function. They are not automatically deployed with a Kubernetes cluster, and can vary in implementation based on intended use, such as load balancing algorithms for Ingress resources. + +[How NGINX Ingress Controller is Designed]({{< relref "overview/design">}}) explains the technical details of the F5 NGINX Ingress Controller. diff --git a/docs/content/includes/index.md b/docs/content/includes/index.md new file mode 100644 index 00000000000..ca03031f1ee --- /dev/null +++ b/docs/content/includes/index.md @@ -0,0 +1,3 @@ +--- +headless: true +--- diff --git a/docs/content/includes/installation/create-common-resources.md b/docs/content/includes/installation/create-common-resources.md new file mode 100644 index 00000000000..6645a3d7985 --- /dev/null +++ b/docs/content/includes/installation/create-common-resources.md @@ -0,0 +1,29 @@ +--- +docs: +--- + +In this section, you'll create resources that most NGINX Ingress Controller installations require: + +1. (Optional) Create a secret for the default NGINX server's TLS certificate and key. Complete this step only if you're using the [default server TLS secret]({{< relref "configuration/global-configuration/command-line-arguments#cmdoption-default-server-tls-secret.md" >}}) command-line argument. If you're not, feel free to skip this step. + + By default, the server returns a _404 Not Found_ page for all requests when no ingress rules are set up. Although we provide a self-signed certificate and key for testing purposes, we recommend using your own certificate. + + To begin, make sure you're in the `kubernetes-ingress/deployment` directory, and then run: + + ```shell + kubectl apply -f ../examples/shared-examples/default-server-secret/default-server-secret.yaml + ``` + +2. Create a ConfigMap to customize your NGINX settings: + + ```shell + kubectl apply -f common/nginx-config.yaml + ``` + +3. Create an `IngressClass` resource. NGINX Ingress Controller won't start without an `IngressClass` resource. + + ```shell + kubectl apply -f common/ingress-class.yaml + ``` + + If you want to make this NGINX Ingress Controller instance your cluster's default, uncomment the `ingressclass.kubernetes.io/is-default-class` annotation. This action will auto-assign `IngressClass` to new ingresses that don't specify an `ingressClassName`. diff --git a/docs/content/includes/installation/manifests/daemonset.md b/docs/content/includes/installation/manifests/daemonset.md new file mode 100644 index 00000000000..050ee649b51 --- /dev/null +++ b/docs/content/includes/installation/manifests/daemonset.md @@ -0,0 +1,21 @@ +--- +docs: +--- + +For additional context on managing containers using Kubernetes DaemonSets, refer to the official Kubernetes [DaemonSets](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) documentation. + +When you deploy NGINX Ingress Controller as a DaemonSet, Kubernetes creates an Ingress Controller pod on every node in the cluster. + +- For NGINX, run: + + ```shell + kubectl apply -f daemon-set/nginx-ingress.yaml + ``` + +- For NGINX Plus, run: + + ```shell + kubectl apply -f daemon-set/nginx-plus-ingress.yaml + ``` + + Update the `nginx-plus-ingress.yaml` file to include your chosen image from the F5 Container registry or your custom container image. diff --git a/docs/content/includes/installation/manifests/deployment.md b/docs/content/includes/installation/manifests/deployment.md new file mode 100644 index 00000000000..45a110081bb --- /dev/null +++ b/docs/content/includes/installation/manifests/deployment.md @@ -0,0 +1,21 @@ +--- +docs: +--- + +For additional context on managing containers using Kubernetes Deployments, refer to the official Kubernetes [Deployments](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) documentation. + +When you deploy NGINX Ingress Controller as a Deployment, Kubernetes automatically sets up a single NGINX Ingress Controller pod. + +- For NGINX, run: + + ```shell + kubectl apply -f deployment/nginx-ingress.yaml + ``` + +- For NGINX Plus, run: + + ```shell + kubectl apply -f deployment/nginx-plus-ingress.yaml + ``` + + Update the `nginx-plus-ingress.yaml` file to include your chosen image from the F5 Container registry or your custom container image. diff --git a/docs/content/includes/installation/manifests/verify-pods-are-running.md b/docs/content/includes/installation/manifests/verify-pods-are-running.md new file mode 100644 index 00000000000..e3edf4e1f5d --- /dev/null +++ b/docs/content/includes/installation/manifests/verify-pods-are-running.md @@ -0,0 +1,9 @@ +--- +docs: +--- + +To confirm the NGINX Ingress Controller pods are operational, run: + +```shell +kubectl get pods --namespace=nginx-ingress +``` diff --git a/docs/content/includes/rbac/set-up-rbac.md b/docs/content/includes/rbac/set-up-rbac.md new file mode 100644 index 00000000000..e17c7687565 --- /dev/null +++ b/docs/content/includes/rbac/set-up-rbac.md @@ -0,0 +1,33 @@ +--- +docs: +--- + +{{}}To complete these steps you need admin access to your cluster. Refer to to your Kubernetes platform's documentation to set up admin access. For Google Kubernetes Engine (GKE), you can refer to their [Role-Based Access Control guide](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control).{{}} + +1. Create a namespace and a service account: + + ```shell + kubectl apply -f common/ns-and-sa.yaml + ``` + +2. Create a cluster role and binding for the service account: + + ```shell + kubectl apply -f rbac/rbac.yaml + ``` + +
+ +If you're planning to use NGINX App Protect or NGINX App Protect DoS, additional roles and bindings are needed. + +1. (NGINX App Protect only) Create the *App Protect* role and binding: + + ```shell + kubectl apply -f rbac/ap-rbac.yaml + ``` + +2. (NGINX App Protect DoS only) Create the *App Protect DoS* role and binding: + + ```shell + kubectl apply -f rbac/apdos-rbac.yaml + ``` diff --git a/docs/content/installation/_index.md b/docs/content/installation/_index.md index ec6d6339722..4ed49bfcbbc 100644 --- a/docs/content/installation/_index.md +++ b/docs/content/installation/_index.md @@ -1,7 +1,7 @@ --- title: Installation description: -weight: 1300 +weight: 400 menu: docs: parent: NGINX Ingress Controller diff --git a/docs/content/installation/building-ingress-controller-image.md b/docs/content/installation/building-ingress-controller-image.md deleted file mode 100644 index 05029d0dd22..00000000000 --- a/docs/content/installation/building-ingress-controller-image.md +++ /dev/null @@ -1,128 +0,0 @@ ---- -title: Building NGINX Ingress Controller Image -description: "This document explains how to build an NGINX Ingress Controller image using the source code." -weight: 2200 -doctypes: [""] -toc: true ---- - -This document explains how to build an NGINX Ingress Controller image using the source code. You can also use pre-built images: please see [here](/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret) and [here](/nginx-ingress-controller/installation/pulling-ingress-controller-image) for details on how to pull the NGINX Ingress Controller based on NGINX Plus from the F5 Docker registry; for NGINX Ingress Controller based on NGINX OSS, we provide the images through [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/) and [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress). - -## Prerequisites - -Before you can build the image, make sure that the following software is installed on your machine: - -- [Docker](https://www.docker.com/products/docker) v19.03+ -- [GNU Make](https://www.gnu.org/software/make/) -- [git](https://git-scm.com/) -- [OpenSSL](https://www.openssl.org/), optionally, if you would like to generate a self-signed certificate and a key for the default server. -- For NGINX Plus, you must have the NGINX Plus license -- the certificate (`nginx-repo.crt`) and the key (`nginx-repo.key`). - -Although the NGINX Ingress Controller is written in golang, golang is not required: you can download the binary file or build the NGINX Ingress Controller in a Docker container. - -## Building the Image and Pushing It to the Private Registry - -We build the image using the make utility and the provided `Makefile`. Let’s create the NGINX Ingress Controller binary, build an image and push the image to the private registry. - -**Note**: If you have a local golang environment and you want to build the binary, you can remove `TARGET=download` from the `make` commands. If you want to build the binary, but you don't have a local golang environment you can use `TARGET=container`. - -1. Make sure to run the `docker login` command first to log in to the registry. - - If you’re using Google Container Registry, make sure you’re logged into the gcloud tool by running the `gcloud auth login` and `gcloud auth configure-docker` commands. - -1. Clone the NGINX Ingress Controller repo: - - ```console - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 - cd kubernetes-ingress - ``` - -1. Build the image: - - For **NGINX**: - - ```console - make debian-image PREFIX=myregistry.example.com/nginx-ingress TARGET=download - ``` - - or if you wish to use alpine - - ```console - make alpine-image PREFIX=myregistry.example.com/nginx-ingress TARGET=download - ``` - - `myregistry.example.com/nginx-ingress` defines the repo in your private registry where the image will be pushed. Substitute that value with the repo in your private registry. - - As a result, the image **myregistry.example.com/nginx-ingress:3.3.2** is built. Note that the tag `3.3.2` comes from the `VERSION` variable, defined in the Makefile. - - - For **NGINX Plus**, first, make sure that the certificate (`nginx-repo.crt`) and the key (`nginx-repo.key`) of your license are located in the root of the project: - - ```console - $ ls nginx-repo.* - nginx-repo.crt nginx-repo.key - ``` - - Then run: - - ```console - make debian-image-plus PREFIX=myregistry.example.com/nginx-plus-ingress TARGET=download - ``` - - `myregistry.example.com/nginx-plus-ingress` defines the repo in your private registry where the image will be pushed. Substitute that value with the repo in your private registry. - - As a result, the image **myregistry.example.com/nginx-plus-ingress:3.3.2** is built. Note that the tag `3.3.2` comes from the `VERSION` variable, defined in the Makefile. - - **Note**: In the event of a patch version of [NGINX Plus being released](/nginx/releases/), make sure to rebuild your image to get the latest version. If your system is caching the Docker layers and not updating the packages, add `DOCKER_BUILD_OPTIONS="--pull --no-cache"` to the `make` command. - -1. Push the image: - - ```console - make push PREFIX=myregistry.example.com/nginx-ingress - ``` - - **Note**: If you're using a different tag, append `TAG=your-tag` to the command above. - -Next you will find the details about available Makefile targets and variables. - -### Makefile Targets - -You can see a list of all the targets by running `make` without any target or `make help` - -Below you can find some of the most useful targets in the **Makefile**: - -- **build**: creates the NGINX Ingress Controller binary using the local golang environment (ignored when `TARGET` is `container`). -- **alpine-image**: for building an alpine-based image with NGINX. -- **alpine-image-plus**: for building an alpine-based image with NGINX Plus. -- **alpine-image-plus-fips**: for building an alpine-based image with NGINX Plus and FIPS inside. -- **debian-image**: for building a debian-based image with NGINX. -- **debian-image-plus**: for building a debian-based image with NGINX Plus. -- **debian-image-nap-plus**: for building a debian-based image with NGINX Plus and the [NGINX App Protect WAF](/nginx-app-protect/) module. -- **debian-image-dos-plus**: for building a debian-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module. -- **debian-image-nap-dos-plus**: for building a debian-based image with NGINX Plus, [NGINX App Protect WAF](/nginx-app-protect/) and [NGINX App Protect DoS](/nginx-app-protect-dos/) modules. -- **ubi-image**: for building an ubi-based image with NGINX for [Openshift](https://www.openshift.com/) clusters. -- **ubi-image-plus**: for building an ubi-based image with NGINX Plus for [Openshift](https://www.openshift.com/) clusters. -- **ubi-image-nap-plus**: for building an ubi-based image with NGINX Plus and the [NGINX App Protect WAF](/nginx-app-protect/) module for [Openshift](https://www.openshift.com/) clusters. -- **ubi-image-dos-plus**: for building an ubi-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module for [Openshift](https://www.openshift.com/) clusters. -- **ubi-image-nap-dos-plus**: for building an ubi-based image with NGINX Plus, [NGINX App Protect WAF](/nginx-app-protect/) and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module for [Openshift](https://www.openshift.com/) clusters. -Note: You need to store your RHEL organization and activation keys in a file named `rhel_license` in the project root. Example: - - ```console - RHEL_ORGANIZATION=1111111 - RHEL_ACTIVATION_KEY=your-key - ``` - -A few other useful targets: - -- **push**: pushes the image to the Docker registry specified in `PREFIX` and `TAG` variables. -- **all**: executes test `test`, `lint`, `verify-codegen`, `update-crds` and `debian-image`. If one of the targets fails, the execution process stops, reporting an error. -- **test**: runs unit tests. -- **certificate-and-key**: The Ingress Controller requires a certificate and a key for the default HTTP/HTTPS server. You can reference them in a TLS Secret in a command-line argument to the Ingress Controller. As an alternative, you can add a file in the PEM format with your certificate and key to the image as `/etc/nginx/secrets/default`. Optionally, you can generate a self-signed certificate and a key using this target. Note that you must add the `ADD` instruction in the Dockerfile to copy the cert and the key to the image. - -### Makefile Variables - -The **Makefile** contains the following main variables for you to customize (either by changing the Makefile or by overriding the variables in the make command): - -- **ARCH** -- the architecture of the image (and the binary). The default value is `amd64`. The most common architectures are `amd64` and `arm64`. Some other popular options are `arm`, `ppc64le` and `s390x`. -- **PREFIX** -- the name of the image. The default is `nginx/nginx-ingress`. -- **TAG** -- the tag added to the image. It's set to the version of the Ingress Controller by default. -- **DOCKER_BUILD_OPTIONS** -- the [options](https://docs.docker.com/engine/reference/commandline/build/#options) for the `docker build` command. For example, `--pull`. -- **TARGET** -- by default, the Ingress Controller is compiled locally using a `local` golang environment. If you want to compile the Ingress Controller using your local golang environment, make sure that the Ingress Controller repo is in your `$GOPATH`. To compile the Ingress Controller using the Docker [golang](https://hub.docker.com/_/golang/) container, specify `TARGET=container`. If you checked out a tag or are on the latest commit on `main` you can specify `TARGET=download` to avoid compiling the binary. diff --git a/docs/content/installation/building-nginx-ingress-controller.md b/docs/content/installation/building-nginx-ingress-controller.md new file mode 100644 index 00000000000..a3d6374e6d0 --- /dev/null +++ b/docs/content/installation/building-nginx-ingress-controller.md @@ -0,0 +1,194 @@ +--- +title: Building NGINX Ingress Controller +description: "Learn how to build an NGINX Ingress Controller image from source codes and upload it to a private Docker registry. You'll also find information on the Makefile targets and variables." +weight: 200 +doctypes: ["installation"] +toc: true +--- + +{{}} + +{{}}If you'd rather not build your own NGINX Ingress Controller image, see the [pre-built image options](#pre-built-images) at the end of this guide.{{}} + +## Before you start + +To get started, you need the following software installed on your machine: + +- [Docker v19.03 or higher](https://docs.docker.com/engine/release-notes/19.03/) +- [GNU Make](https://www.gnu.org/software/make/) +- [git](https://git-scm.com/) +- [OpenSSL](https://www.openssl.org/), optionally, if you would like to generate a self-signed certificate and a key for the default server. +- For NGINX Plus users, download the certificate (_nginx-repo.crt_) and key (_nginx-repo.key_) from [MyF5](https://my.f5.com). + +Although NGINX Ingress Controller is written in Golang, you don't need to have Golang installed. You can either download the precompiled binary file or build NGINX Ingress Controller in a Docker container. + +--- + +## Prepare the environment {#prepare-environment} + +Get your system ready for building and pushing the NGINX Ingress Controller image. + +1. Sign in to your private registry. Replace `` with the path to your own private registry. + + ```shell + docker login + ``` + +2. Clone the NGINX Ingress Controller GitHub repository. Replace `` with the version of NGINX Ingress Controller you want. + + ```shell + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch + cd kubernetes-ingress + ``` + + For instance if you want to clone version v3.2.1, the commands to run would be: + + ```shell + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1 + cd kubernetes-ingress + ``` + +--- + +## Build the NGINX Ingress Controller image {#build-image} + +After setting up your environment, follow these steps to build the NGINX Ingress Controller image. + +{{}}If you have a local Golang environment and want to build the binary yourself, remove `TARGET=download` from the make commands. If you don't have Golang but still want to build the binary, use `TARGET=container`.{{}} + +### For NGINX + +1. Build the image. Replace `` with your private registry's path. + + - For a Debian-based image: + + ```shell + make debian-image PREFIX=/nginx-ingress TARGET=download + ``` + + - For an Alpine-based image: + + ```shell + make alpine-image PREFIX=/nginx-ingress TARGET=download + ``` + + **What to expect**: The image is built and tagged with a version number, which is derived from the `VERSION` variable in the [_Makefile_](#makefile-details). This version number is used for tracking and deployment purposes. + +### For NGINX Plus + +1. Place your NGINX Plus license files (_nginx-repo.crt_ and _nginx-repo.key_) in the project's root folder. To verify they're in place, run: + + ```shell + ls nginx-repo.* + ``` + + You should see: + + ```shell + nginx-repo.crt nginx-repo.key + ``` + +2. Build the image. Replace `` with your private registry's path. + + ```shell + make debian-image-plus PREFIX=/nginx-plus-ingress TARGET=download + ``` + +
+ + **What to expect**: The image is built and tagged with a version number, which is derived from the `VERSION` variable in the [_Makefile_](#makefile-details). This version number is used for tracking and deployment purposes. + +{{}}In the event a patch version of NGINX Plus is released, make sure to rebuild your image to get the latest version. If your system is caching the Docker layers and not updating the packages, add `DOCKER_BUILD_OPTIONS="--pull --no-cache"` to the make command.{{}} + +--- + +## Push the image to your private registry {#push-image} + +Once you've successfully built the NGINX or NGINX Plus Ingress Controller image, the next step is to upload it to your private Docker registry. This makes the image available for deployment to your Kubernetes cluster. + +### For NGINX + +1. Upload the NGINX image. If you're using a custom tag, append `TAG=your-tag` to the command. Replace `` with your private registry's path. + + ```shell + make push PREFIX=/nginx-ingress + ``` + +### For NGINX Plus + +1. Upload the NGINX Plus image. Like with the NGINX image, if you're using a custom tag, add `TAG=your-tag` to the end of the command. Replace `` with your private registry's path. + + ```shell + make push PREFIX=/nginx-plus-ingress + ``` + +--- + +## Makefile details {#makefile-details} + +This section provides comprehensive information on the targets and variables available in the _Makefile_. These targets and variables allow you to customize how you build, tag, and push your NGINX or NGINX Plus images. + +### Key Makefile targets {#key-makefile-targets} + +{{}}To view available _Makefile_ targets, run `make` with no target or type `make help`.{{}} + +Key targets include: + +{{}} +|
Target | Description | +|---------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| _build_ | Creates the NGINX Ingress Controller binary with your local Go environment. | +| _alpine-image_ | Builds an Alpine-based image with NGINX. | +| _alpine-image-plus_ | Builds an Alpine-based image with NGINX Plus. | +| _alpine-image-plus-fips_ | Builds an Alpine-based image with NGINX Plus and FIPS. | +| _debian-image_ | Builds a Debian-based image with NGINX. | +| _debian-image-plus_ | Builds a Debian-based image with NGINX Plus. | +| _debian-image-nap-plus_ | Builds a Debian-based image with NGINX Plus and the [NGINX App Protect WAF](/nginx-app-protect/) module. | +| _debian-image-dos-plus_ | Builds a Debian-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module. | +| _debian-image-nap-dos-plus_ | Builds a Debian-based image with NGINX Plus, [NGINX App Protect WAF](/nginx-app-protect/) and [NGINX App Protect DoS](/nginx-app-protect-dos/) modules. | +| _ubi-image_ | Builds a UBI-based image with NGINX for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-plus_ | Builds a UBI-based image with NGINX Plus for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-nap-plus_ | Builds a UBI-based image with NGINX Plus and the [NGINX App Protect WAF](/nginx-app-protect/) module for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-dos-plus_ | Builds a UBI-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module for [OpenShift](https://www.openshift.com/) clusters. | +| _ubi-image-nap-dos-plus_ |

Builds a UBI-based image with NGINX Plus, [NGINX App Protect WAF](/nginx-app-protect/) and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module for [OpenShift](https://www.openshift.com/) clusters.

**Important**: Save your RHEL organization and activation keys in a file named _rhel_license_ at the project root.

For instance:

RHEL_ORGANIZATION=1111111
RHEL_ACTIVATION_KEY=your-key
| +{{}} + +### Additional useful targets {#other-makefile-targets} + +A few other useful targets: + +{{}} +|
Target
| Description | +|---------------------------------------|---------------| +| _push_ | Pushes the built image to the Docker registry. Configures with `PREFIX` and `TAG`. | +| _all_ | Runs `test`, `lint`, `verify-codegen`, `update-crds`, and `debian-image`. Stops and reports an error if any of these targets fail. | +| _test_ | Runs unit tests. | +| _certificate-and-key_ | NGINX Ingress Controller requires a certificate and key for the default HTTP/HTTPS server. You have several options:
  • Reference them in a TLS Secret in a command-line argument to NGINX Ingress Controller.
  • Add them to the image in in a file in PEM format as `/etc/nginx/secrets/default`.
  • Generate a self-signed certificate and key with this target.
Note, you must include the `ADD` instruction in your Dockerfile to copy the cert and key to the image. | +{{}} + +### Makefile variables you can customize {#makefile-variables} + +The _Makefile_ includes several key variables. You have the option to either modify these variables directly in the _Makefile_ or override them when you run the `make` command. + +{{}} +|
Variable
| Description | +|-----------------------------------------|---------------| +| _ARCH_ | Defines the architecture for the image and binary. The default is `amd64`, but you can also choose from `arm64`, `arm`, `ppc64le`, and `s390x`. | +| _PREFIX_ | Gives the image its name. The default is `nginx/nginx-ingress`. | +| _TAG_ | Adds a tag to the image. This is often the version of the NGINX Ingress Controller. | +| _DOCKER\_BUILD\_OPTIONS_ | Allows for additional [options](https://docs.docker.com/engine/reference/commandline/build/#options) during the `docker build` process, like `--pull`. | +| _TARGET_ |

Determines the build environment. NGINX Ingress Controller compiles locally in a Golang environment by default. Ensure the NGINX Ingress Controller repo resides in your `$GOPATH` if you select this option.

Alternatively, you can set `TARGET=container` to build using a Docker [Golang](https://hub.docker.com/_/golang/) container. To skip compiling the binary if you're on a specific tag or the latest `main` branch commit, set `TARGET=download`.

| +{{}} + +--- + +## Alternatives to building your own image {#pre-built-images} + +If you prefer not to build your own NGINX Ingress Controller image, you can use pre-built images. Here are your options: + +**NGINX Ingress Controller**: Download the image `nginx/nginx-ingress` from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress) or [GitHub](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress). + +**NGINX Plus Ingress Controller**: You have two options for this, both requiring an NGINX Ingress Controller subscription. + +- Download the image using your NGINX Ingress Controller subscription certificate and key. See the [Getting the F5 Registry NGINX Ingress Controller Image]({{< relref "installation/nic-images/pulling-ingress-controller-image.md" >}}) guide. +- Use your NGINX Ingress Controller subscription JWT token to get the image: Instructions are in [Getting the NGINX Ingress Controller Image with JWT]({{< relref "installation/nic-images/using-the-jwt-token-docker-secret.md" >}}). diff --git a/docs/content/installation/ingress-nginx.md b/docs/content/installation/ingress-nginx.md new file mode 100644 index 00000000000..deac49b786d --- /dev/null +++ b/docs/content/installation/ingress-nginx.md @@ -0,0 +1,552 @@ +--- +title: "Migrating from Ingress-NGINX Controller to NGINX Ingress Controller" +date: 2023-09-29T16:31:21+01:00 +description: "This document describes how to migrate from the community-maintained Ingress-NGINX Controller to the F5 NGINX Ingress Controller." +weight: 500 +toc: true +tags: [ "docs" ] +docs: "DOCS-000" +categories: ["installation", "platform management"] +doctypes: ["tutorial"] +journeys: ["getting started"] +personas: ["devops"] +authors: ["Jason Williams"] +--- + +
+ +## Overview + +This page explains two different ways to migrate from the community-maintained [Ingress-NGINX Controller](https://github.com/kubernetes/ingress-nginx) project to NGINX Ingress Controller: using NGINX's Ingress Resources or with Kubernetes's built-in Ingress Resources. This is typically because of implementation differences, and to take advantage of features such as [NGINX Plus integration]({{}}). + + + +The information in this guide is extracted from a free eBook called "_Kubernetes Ingress Controller Deployment and Security with NGINX_", which can be downloaded from the [NGINX Library](https://www.nginx.com/resources/library/kubernetes-ingress-controller-deployment-security-nginx/). + +## Before you begin + +To complete the instructions in this guide, you need the following: + +- A working knowledge of [Ingress Controllers]({{}}). +- An [NGINX Ingress Controller installation]({{}}) on the same host as an existing Ingress-NGINX Controller. + +There are two primary paths for migrating between the community Ingress-NGINX Controller to NGINX Ingress Controller: + +- Using NGINX Ingress Resources +- Using Kubernetes Ingress Resources. + +## Migration with NGINX Ingress resources +This path uses Kubernetes Ingress Resources to set root permissions, then NGINX Ingress Resources for configuration using custom resource definitions (CRDs): + +* [VirtualServer and VirtualServerRoute]({{}}) +* [TransportServer]({{}}) +* [GlobalConfiguration]({{}}) +* [Policy]({{}}) + +### Configuring SSL termination and HTTP path-based routing +The following two code examples correspond to a Kubernetes Ingress Resource and an [NGINX VirtualServer Resource]({{}}). Although the syntax and indentation is different, they accomplish the same basic Ingress functions, used for SSL termination and Layer 7 path-based routing. + +**Kubernetes Ingress Resource** +```yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: nginx-test +spec: + tls: + - hosts: + - foo.bar.com + secretName: tls-secret + rules: + - host: foo.bar.com + http: + paths: + - path: /login + backend: + serviceName: login-svc + servicePort: 80 + - path: /billing + serviceName: billing-svc + servicePort: 80 +``` + +**NGINX VirtualServer Resource** +```yaml +apiVersion: networking.k8s.io/v1 +kind: VirtualServer +metadata: + name: nginx-test +spec: + host: foo.bar.com + tls: + secret: tls-secret + upstreams: + - name: login + service: login-svc + port: 80 + - name: billing + service: billing-svc + port: 80 + routes: + - path: /login + action: + pass: login + - path: /billing + action: + pass: billing +``` + +### Configuring TCP/UDP load balancing and TLS passthrough +NGINX Ingress Controller exposes TCP and UDP services using [TransportServer]({{}}) and [GlobalConfiguration]({{}}) resources. These resources provide a broad range of options for TCP/UDP and TLS Passthrough load balancing. By contrast, the community Ingress-NGINX Controller exposes TCP/UDP services by using a Kubernetes ConfigMap object. + +--- + +### Convert Ingress-NGINX Controller annotations to NGINX Ingress resources +Kubernetes deployments often need to extend basic Ingress rules for advanced use cases such as canary and blue-green deployments, traffic throttling, and ingress-egress traffic manipulation. The community Ingress-NGINX Controller implements many of these using Kubernetes annotations with custom Lua extensions. + +These custom Lua extensions are intended for specific NGINX Ingress resource definitions and may not be as granular as required for advanced use cases. The following examples show how to convert these annotations into NGINX Ingress Controller Resources. + +--- + +#### Canary deployments +Canary and blue-green deployments allow you to push code changes to production environments without disrupting existing users. NGINX Ingress Controller runs them on the data plane: to migrate from the community Ingress-NGINX Controller, you must map the latter's annotations to [VirtualServer and VirtualServerRoute resources]({{}}). + +The Ingress-NGINX Controller evaluates canary annotations in the following order: + +1. _nginx.ingress.kubernetes.io/canary-by-header_ +1. _nginx.ingress.kubernetes.io/canary-by-cookie_ +1. _nginx.ingress.kubernetes.io/canary-by-weight_ + +For NGINX Ingress Controller to evalute them the same way, they must appear in the same order in the VirtualServer or VirtualServerRoute Manifest. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/canary: "true" +nginx.ingress.kubernetes.io/canary-by-header: "httpHeader" +``` + +**NGINX Ingress Controller** +```yaml +matches: +- conditions: + - header: httpHeader + value: never + action: + pass: echo + - header: httpHeader + value: always + action: + pass: echo-canary +action: + pass: echo +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/canary: "true" +nginx.ingress.kubernetes.io/canary-by-header: "httpHeader" +nginx.ingress.kubernetes.io/canary-by-header-value: "my-value" +``` + +**NGINX Ingress Controller** +```yaml +matches: +- conditions: + - header: httpHeader + value: my-value + action: + pass: echo-canary +action: + pass: echo +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/canary: "true" +nginx.ingress.kubernetes.io/canary-by-cookie: "cookieName" +``` + +**NGINX Ingress Controller** +```yaml +matches: +- conditions: + - cookie: cookieName + value: never + action: + pass: echo + - cookie: cookieName + value: always + action: + pass: echo-canary +action: + pass: echo +``` + +--- + +#### Traffic control +Environments using microservices tend to use extensive traffic-control policies to manage ephemeral applications using circuit breaking and rate and connection limiting to prevent error conditions due to unhealthy states or abnormal behavior. + +The following examples map Ingress-NGINX Controller annotations to NGINX [VirtualServer and VirtualServerRoute resources]({{}}) for rate limiting, custom HTTP errors, custom default backend and URI rewriting. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/custom-http-errors: "code" + +nginx.ingress.kubernetes.io/default-backend: "default-svc" +``` + +**NGINX Ingress Controller** +```yaml +errorPages: +- codes: [code] + redirect: + code: 301 + url: default-svc +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-connections: "number" +``` + +**NGINX Ingress Controller** +```yaml +http-snippets: | + limit_conn_zone $binary_remote_addr zone=zone_name:size; +routes: +- path: /path + location-snippets: | + limit_conn zone_name number; +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-rate: "number" +nginx.ingress.kubernetes.io/limit-rate-after: "number" +``` + +**NGINX Ingress Controller** +```yaml +location-snippets: | + limit_rate number; + + limit_rate_after number; +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-rpm: "number" +nginx.ingress.kubernetes.io/limit-burst-multiplier: "multiplier" +``` + +**NGINX Ingress Controller** +```yaml +rateLimit: + rate: numberr/m + + burst: number * multiplier + key: ${binary_remote_addr} + zoneSize: size +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-rps: "number" +nginx.ingress.kubernetes.io/limit-burst-multiplier: "multiplier" +``` + +**NGINX Ingress Controller** +```yaml +rateLimit: + rate: numberr/s + + burst: number * multiplier + key: ${binary_remote_addr} + zoneSize: size +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/limit-whitelist: "CIDR" +``` + +**NGINX Ingress Controller** +```yaml +http-snippets: | +server-snippets: | +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/rewrite-target: "URI" +``` + +**NGINX Ingress Controller** +```yaml +rewritePath: "URI" +``` + +There are four Ingress-NGINX Controller annotations without NGINX Ingress resource fields yet: they must be handled using snippets. + +- _nginx.ingress.kubernetes.io/limit-connections_ +- _nginx.ingress.kubernetes.io/limit-rate_ +- _nginx.ingress.kubernetes.io/limit-rate-after_ +- _nginx.ingress.kubernetes.io/limit-whitelist_ + +--- + +#### Header manipulation +Manipulating HTTP headers is useful in many cases, as they contain information that is important and relevant to systems involved in HTTP transactions. The community Ingress-NGINX Controller supports enabling and configuring cross-origin resource sharing (CORS) headings used by AJAX applications, where front-end Javascript code interacts with backend applications or web servers. + +These code blocks show how the Ingress-NGINX annotations correspond to NGINX Ingress Controller [VirtualServer and VirtualServerRoute resources]({{}}). + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/enable-cors: "true" +nginx.ingress.kubernetes.io/cors-allow-credentials: "true" + +nginx.ingress.kubernetes.io/cors-allow-headers: "X-Forwarded-For" + +nginx.ingress.kubernetes.io/cors-allow-methods: "PUT, GET, POST, OPTIONS" + +nginx.ingress.kubernetes.io/cors-allow-origin: "*" + +nginx.ingress.kubernetes.io/cors-max-age: "seconds" +``` + +**NGINX Ingress Controller** +```yaml +responseHeaders: + add: + - name: Access-Control-Allow-Credentials + value: "true" + - name: Access-Control-Allow-Headers + value: "X-Forwarded-For" + - name: Access-Control-Allow-Methods + value: "PUT, GET, POST, OPTIONS" + - name: Access-Control-Allow-Origin + value: "*" + - name: Access-Control-Max-Age + value: "seconds" +``` + +--- + +#### Proxying and load balancing +NGINX Ingress Controller has multiple proxy and load balancing functionalities you may want to configure based on the use case, such as configuring the load balancing algorithm and the timeout and buffering settings for proxied connections. + +This table shows how Ingress-NGINX Controller annotations map to statements in the upstream field for [VirtualServer and VirtualServerRoute resources]({{}}), covering load balancing, proxy timeout, proxy buffering and connection routing for a services' ClusterIP address and port. + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| Ingress-NGINX Controller | NGINX Ingress Controller | +| ------------------------ | ------------------------ | +| _nginx.ingress.kubernetes.io/load-balance_ | _lb-method_ | +| _nginx.ingress.kubernetes.io/proxy-buffering_ | _buffering_ | +| _nginx.ingress.kubernetes.io/proxy-buffers-number_ | _buffers_ | +| _nginx.ingress.kubernetes.io/proxy-buffer-size_ | _buffers_ | +| _nginx.ingress.kubernetes.io/proxy-connect-timeout_ | _connect-timeout_ | +| _nginx.ingress.kubernetes.io/proxy-next-upstream_ | _next-upstream_ | +| _nginx.ingress.kubernetes.io/proxy-next-upstream-timeout_ | _next-upstream-timeout_ | +| _nginx.ingress.kubernetes.io/proxy-read-timeout_ | _read-timeout_ | +| _nginx.ingress.kubernetes.io/proxy-send-timeout_ | _send-timeout_ | +| _nginx.ingress.kubernetes.io/service-upstream_ | _use-cluster-ip_ | +{{% /bootstrap-table %}} + +#### mTLS authentication + +mTLS authentication is a way of enforcing mutual authentication on traffic entering and exiting a cluster (north-sourth traffic). This secure form of communication is common within a service mesh, commonly used in strict zero-trust environments. + +NGINX Ingress Controller layer can handle mTLS authentication for end systems through the presentation of valid certificates for external connections. It accomplishes this through [Policy]({{}}) resources, which correspond to Ingress-NGINX Controller annotations for [client certificate authentication](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#client-certificate-authentication) and [backend certificate authentication](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#backend-certificate-authentication). + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/auth-tls-secret: secretName +nginx.ingress.kubernetes.io/auth-tls-verify-client: "on" +nginx.ingress.kubernetes.io/auth-tls-verify-depth: "1" +``` + +**NGINX Ingress Controller** +```yaml +ingressMTLS: + clientCertSecret: secretName + verifyClient: "on" + + verifyDepth: 1 +``` + +--- + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/proxy-ssl-secret: "secretName" +nginx.ingress.kubernetes.io/proxy-ssl-verify: "on|off" +nginx.ingress.kubernetes.io/proxy-ssl-verify-depth: "1" +nginx.ingress.kubernetes.io/proxy-ssl-protocols: "TLSv1.2" +nginx.ingress.kubernetes.io/proxy-ssl-ciphers: "DEFAULT" +nginx.ingress.kubernetes.io/proxy-ssl-name: "server-name" +nginx.ingress.kubernetes.io/proxy-ssl-server-name: "on|off" +``` + +**NGINX Ingress Controller** +```yaml +egressMTLS: + tlsSecret: secretName + + verifyServer: true|false + + verifyDepth: 1 + + protocols: TLSv1.2 + + ciphers: DEFAULT + + sslName: server-name + + serverName: true|false +``` + +--- + +#### Session persistence with NGINX Plus +With [NGINX Plus]({{}}), you can use [Policy]({{}}) resources for session persistence, which have corresponding annotations for the community Ingress-NGINX Controller. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/affinity: "cookie" +nginx.ingress.kubernetes.io/session-cookie-name: "cookieName" +nginx.ingress.kubernetes.io/session-cookie-expires: "x" +nginx.ingress.kubernetes.io/session-cookie-path: "/route" +nginx.ingress.kubernetes.io/session-cookie-secure: "true" +``` + +**NGINX Ingress Controller** +```yaml +sessionCookie: + enable: true + + name: cookieName + + expires: xh + + path: /route + + secure: true +``` + +## Migration with Kubernetes Ingress resources +The other option for migrating from the community Ingress-NGINX Controller to NGINX Ingress Controller is using only [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) and [ConfigMaps](https://kubernetes.io/docs/concepts/configuration/configmap/) from standard Kubernetes resources, potentially relying on [mergeable Ingress types](https://github.com/nginxinc/kubernetes-ingress/tree/main/examples/ingress-resources/mergeable-ingress-types). + +This ensures that all configuration is kept in the Ingress object. + +{{< warning >}} +You should avoid altering the `spec` field of the Ingress resource when taking this option. Ingress-NGINX Controller and NGINX Ingress Controller differ slightly in their implementations: changing the Kubernetes Ingress can create incompatibility issues. +{{< /warning >}} + +### Advanced configuration with annotations +This table maps the Ingress-NGINX Controller annotations to NGINX Ingress Controller's equivalent annotations, and the respective NGINX Directive. + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| Ingress-NGINX Controller | NGINX Ingress Controller | NGINX Directive | +| ------------------------ | ------------------------ | --------------- | +| [_nginx.ingress.kubernetes.io/configuration-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#configuration-snippet) | [_nginx.org/location-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#snippets-and-custom-templates) | N/A | +| [_nginx.ingress.kubernetes.io/load-balance_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-nginx-load-balancing) (1) | [_nginx.org/lb-method_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#backend-services-upstreams) | [_random two least_conn_](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#random) | +| [_nginx.ingress.kubernetes.io/proxy-buffering_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#proxy-buffering) | [_nginx.org/proxy-buffering_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_buffering_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffering) | +| [_nginx.ingress.kubernetes.io/proxy-buffers-number_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#proxy-buffers-number) | [_nginx.org/proxy-buffers_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_buffers_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffers) | +| [_nginx.ingress.kubernetes.io/proxy-buffer-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#proxy-buffer-size) | [_nginx.org/proxy-buffer-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_buffer_size_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_buffer_size) | +| [_nginx.ingress.kubernetes.io/proxy-connect-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-timeouts) | [_nginx.org/proxy-connect-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_connect_timeout_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_connect_timeout) | +| [_nginx.ingress.kubernetes.io/proxy-read-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-timeouts) | [_nginx.org/proxy-read-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_read_timeout_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_read_timeout) | +| [_nginx.ingress.kubernetes.io/proxy-send-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#custom-timeouts) | [_nginx.org/proxy-send-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#general-customization) | [_proxy_send_timeout_](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_send_timeout) | +| [_nginx.ingress.kubernetes.io/rewrite-target_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#rewrite) | [_nginx.org/rewrites_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#request-uriheader-manipulation) | [_rewrite_](https://nginx.org/en/docs/http/ngx_http_rewrite_module.html#rewrite) | +| [_nginx.ingress.kubernetes.io/server-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#server-snippet)| [_nginx.org/server-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#snippets-and-custom-templates) | N/A | +| [_nginx.ingress.kubernetes.io/ssl-redirect_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#server-side-https-enforcement-through-redirect) | [_ingress.kubernetes.io/ssl-redirect_](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#auth-and-ssltls) | N/A (2) | +{{% /bootstrap-table %}} + +1. Ingress-NGINX Controller implements some of its load balancing algorithms with Lua, which may not have an equivalent in NGINX Ingress Controller. +1. To redirect HTTP (80) traffic to HTTPS (443), NGINX Ingress Controller uses native NGINX `if` conditions while Ingress-NGINX Controller uses Lua. + +The following two snippets outline Ingress-NGINX Controller annotations that correspond to annotations for NGINX Ingress Controller with NGINX Plus. + +**Ingress-NGINX Controller** +```yaml +nginx.ingress.kubernetes.io/affinity: "cookie" +nginx.ingress.kubernetes.io/session-cookie-name: "cookie_name" +nginx.ingress.kubernetes.io/session-cookie-expires: "seconds" +nginx.ingress.kubernetes.io/session-cookie-path: "/route" +``` + +**NGINX Ingress Controller (with NGINX Plus)** +```yaml +nginx.com/sticky-cookie-services: "serviceName=example-svc cookie_name expires=time path=/route" +``` + +{{< note >}} +NGINX Ingress Controller has additional annotations for features using NGINX Plus that have no Ingress-NGINX Controller equivalent, such as active health checks and authentication using JSON Web Tokens (JWTs). +{{< /note >}} + +### Global configuration with ConfigMaps + +This table maps the Ingress-NGINX Controller ConfigMap keys to NGINX Ingress Controller's equivalent ConfigMap keys. + + + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| Ingress-NGINX Controller | NGINX Ingress Controller | +| ------------------------ | ------------------------ | +| [_disable-access-log_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#disable-access-log) | [_access-log-off_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_error-log-level_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#error-log-level) | [_error-log-level_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_hsts_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts) | [_hsts_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_hsts-include-subdomains_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts-include-subdomains) | [_hsts-include-subdomains_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_hsts-max-age_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#hsts-max-age) | [_hsts-max-age_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_http-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#http-snippet) | [_http-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_keep-alive_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive) | [_keepalive-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_keep-alive-requests_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#keep-alive-requests) | [_keepalive-requests_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_load-balance_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#load-balance) | [_lb-method_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#backend-services-upstreams) | +| [_location-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#location-snippet) | [_location-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_log-format-escape-json_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#log-format-escape-json) | [_log-format-escaping: "json"_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_log-format-stream_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#log-format-stream) | [_stream-log-format_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_log-format-upstream_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#log-format-upstream) | [_log-format_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#logging) | +| [_main-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#main-snippet) | [_main-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_max-worker-connections_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-connections) | [_worker-connections_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_max-worker-open-files_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#max-worker-open-files) | [_worker-rlimit-nofile_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-body-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-body-size) | [_client-max-body-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-buffering_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-buffering) | [_proxy-buffering_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-buffers-number_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-buffers-number) | [_proxy-buffers: number size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-buffer-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-buffer-size) | [_proxy-buffers: number size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-connect-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-connect-timeout) | [_proxy-connect-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-read-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-read-timeout) | [_proxy-read-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-send-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-send-timeout) | [_proxy-send-timeout_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_server-name-hash-bucket-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#server-name-hash-bucket-size) | [_server-names-hash-bucket-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_proxy-headers-hash-max-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#proxy-headers-hash-max-size) | [_server-names-hash-max-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_server-snippet_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#server-snippet) | [_server-snippets_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#snippets-and-custom-templates) | +| [_server-tokens _](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#server-tokens) | [_server-tokens_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_ssl-ciphers_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-ciphers) | [_ssl-ciphers_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_ssl-dh-param_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-dh-param) | [_ssl-dhparam-file_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_ssl-protocols_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-protocols) | [_ssl-protocols_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_ssl-redirect_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#ssl-redirect) | [_ssl-redirect_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#auth-and-ssltls) | +| [_upstream-keepalive-connections_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#upstream-keepalive-connections) | [_keepalive_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#backend-services-upstreams) | +| [_use-http2_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-http2) | [_http2_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#listeners) | +| [_use-proxy-protocol_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#use-proxy-protocol) | [_proxy-protocol_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#listeners) | +| [_variables-hash-bucket-size_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#variables-hash-bucket-size) | [_variables-hash-bucket-size_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_worker-cpu-affinity_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-cpu-affinity) | [_worker-cpu-affinity_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_worker-processes_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-processes) | [_worker-processes_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +| [_worker-shutdown-timeout_](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/#worker-shutdown-timeout) | [_worker-shutdown-timeole_](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#general-customization) | +{{% /bootstrap-table %}} diff --git a/docs/content/installation/installation-with-helm.md b/docs/content/installation/installation-with-helm.md deleted file mode 100644 index 6407301529e..00000000000 --- a/docs/content/installation/installation-with-helm.md +++ /dev/null @@ -1,489 +0,0 @@ ---- -title: Installation with Helm -description: This document describes how to install the NGINX Ingress Controller in your Kubernetes cluster using Helm. -weight: 1900 -doctypes: [""] -toc: true -docs: "DOCS-602" ---- - - -## Introduction - -This chart deploys the NGINX Ingress Controller in your Kubernetes cluster. - -## Prerequisites - -**Note** All documentation should only be used with the latest stable release, indicated on -[the releases page](https://github.com/nginxinc/kubernetes-ingress/releases) of the GitHub repository. - -- A [Kubernetes Version Supported by the Ingress - Controller](https://docs.nginx.com/nginx-ingress-controller/technical-specifications/#supported-kubernetes-versions) -- Helm 3.0+. -- If you’d like to use NGINX Plus: - - To pull from the F5 Container registry, configure a docker registry secret using your JWT token from the MyF5 portal - by following the instructions from - [here](https://docs.nginx.com/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret). Make sure to - specify the secret using `controller.serviceAccount.imagePullSecretName` parameter. - - Alternatively, pull an Ingress Controller image with NGINX Plus and push it to your private registry by following - the instructions from - [here](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image). - - Alternatively, you can build an Ingress Controller image with NGINX Plus and push it to your private registry by - following the instructions from - [here](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image). - - Update the `controller.image.repository` field of the `values-plus.yaml` accordingly. -- If you’d like to use App Protect DoS, please install App Protect DoS Arbitrator [helm - chart](https://github.com/nginxinc/nap-dos-arbitrator-helm-chart). Make sure to install in the same namespace as the - NGINX Ingress Controller. Note that if you install multiple NGINX Ingress Controllers in the same namespace, they will - need to share the same Arbitrator because it is not possible to install more than one Arbitrator in a single - namespace. - -## CRDs - -By default, the Ingress Controller requires a number of custom resource definitions (CRDs) installed in the cluster. The -Helm client will install those CRDs. If the CRDs are not installed, the Ingress Controller pods will not become `Ready`. - -If you do not use the custom resources that require those CRDs (which corresponds to `controller.enableCustomResources` -set to `false` and `controller.appprotect.enable` set to `false` and `controller.appprotectdos.enable` set to `false`), -the installation of the CRDs can be skipped by specifying `--skip-crds` for the helm install command. - -### Upgrading the CRDs - -To upgrade the CRDs, pull the chart sources as described in [Pulling the Chart](#pulling-the-chart) and then run: - -```console -kubectl apply -f crds/ -``` - -> **Note** -> -> The following warning is expected and can be ignored: `Warning: kubectl apply should be used on resource created by -> either kubectl create --save-config or kubectl apply`. -> -> Make sure to check the [release notes](https://www.github.com/nginxinc/kubernetes-ingress/releases) for a new release -> for any special upgrade procedures. - -### Uninstalling the CRDs - -To remove the CRDs, pull the chart sources as described in [Pulling the Chart](#pulling-the-chart) and then run: - -```console -kubectl delete -f crds/ -``` - -> **Note** -> -> This command will delete all the corresponding custom resources in your cluster across all namespaces. Please ensure -> there are no custom resources that you want to keep and there are no other Ingress Controller releases running in the -> cluster. - -## Managing the Chart via OCI Registry - -### Installing the Chart - -To install the chart with the release name my-release (my-release is the name that you choose): - -For NGINX: - -```console -helm install my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 1.0.2 -``` - -For NGINX Plus: (assuming you have pushed the Ingress Controller image `nginx-plus-ingress` to your private registry -`myregistry.example.com`) - -```console -helm install my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 1.0.2 --set controller.image.repository=myregistry.example.com/nginx-plus-ingress --set controller.nginxplus=true -``` - -This will install the latest `edge` version of the Ingress Controller from GitHub Container Registry. If you prefer to -use Docker Hub, you can replace `ghcr.io/nginxinc/charts/nginx-ingress` with -`registry-1.docker.io/nginxcharts/nginx-ingress`. - -### Upgrading the Chart - -Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, see [Upgrading the -CRDs](#upgrading-the-crds). - -To upgrade the release `my-release`: - -```console -helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 1.0.2 -``` - -### Uninstalling the Chart - -To uninstall/delete the release `my-release`: - -```console -helm uninstall my-release -``` - -The command removes all the Kubernetes components associated with the release and deletes the release. - -Uninstalling the release does not remove the CRDs. To remove the CRDs, see [Uninstalling the -CRDs](#uninstalling-the-crds). - -### Edge Version - -To test the latest changes in NGINX Ingress Controller before a new release, you can install the `edge` version. This -version is built from the `main` branch of the NGINX Ingress Controller repository. You can install the `edge` version -by specifying the `--version` flag with the value `0.0.0-edge`: - -```console -helm install my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.0.0-edge -``` - -> **Warning** -> -> The `edge` version is not intended for production use. It is intended for testing and development purposes only. - -## Managing the Chart via Sources - -### Pulling the Chart - -This step is required if you're installing the chart using its sources. Additionally, the step is also required for -managing the custom resource definitions (CRDs), which the Ingress Controller requires by default, or for -upgrading/deleting the CRDs. - -1. Pull the chart sources: - - ```console - helm pull oci://ghcr.io/nginxinc/charts/nginx-ingress --untar --version 1.0.2 - ``` - -2. Change your working directory to nginx-ingress: - - ```console - cd nginx-ingress - ``` - -### Installing the Chart - -To install the chart with the release name my-release (my-release is the name that you choose): - -For NGINX: - -```console -helm install my-release . -``` - -For NGINX Plus: - -```console -helm install my-release -f values-plus.yaml . -``` - -The command deploys the Ingress Controller in your Kubernetes cluster in the default configuration. The configuration -section lists the parameters that can be configured during installation. - -### Upgrading the Chart - -Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, see [Upgrading the -CRDs](#upgrading-the-crds). - -To upgrade the release `my-release`: - -```console -helm upgrade my-release . -``` - -### Uninstalling the Chart - -To uninstall/delete the release `my-release`: - -```console -helm uninstall my-release -``` - -The command removes all the Kubernetes components associated with the release and deletes the release. - -Uninstalling the release does not remove the CRDs. To remove the CRDs, see [Uninstalling the -CRDs](#uninstalling-the-crds). - -## Upgrading without downtime - -### Background - -In NGINX Ingress Controller version 3.1.0, [changes were introduced](https://github.com/nginxinc/kubernetes-ingress/pull/3606) -to Helm resource names, labels and annotations to fit with Helm best practices. -When using Helm to upgrade from a version prior to 3.1.0, certain resources like Deployment, DaemonSet and Service will -be recreated due to the aforementioned changes, which will result in downtime. - -Although the advisory is to update all resources in accordance with new naming convention, to avoid the downtime -please follow the steps listed in this page. - -### Upgrade Steps - -**Note** The following steps apply to both 2.x and 3.0.x releases. - -The steps you should follow depend on the Helm release name: - -{{}} - -{{%tab name="Helm release name is `nginx-ingress`"%}} - -1. Use `kubectl describe` on deployment/daemonset to get the `Selector` value: - - ```shell - kubectl describe deployments -n - ``` - - Copy the key=value under `Selector`, such as: - - ```shell - Selector: app=nginx-ingress-nginx-ingress - ``` - -2. Checkout the latest available tag using `git checkout v3.3.2` - -3. Navigate to `/kubernates-ingress/deployments/helm-chart` - -4. Update the `selectorLabels: {}` field in the `values.yaml` file located at `/kubernates-ingress/deployments/helm-chart` -with the copied `Selector` value. - - ```shell - selectorLabels: {app: nginx-ingress-nginx-ingress} - ``` - -5. Run `helm upgrade` with following arguments set: - - ```shell - --set serviceNameOverride="nginx-ingress-nginx-ingress" - --set controller.name="" - --set fullnameOverride="nginx-ingress-nginx-ingress" - ``` - - It could look as follows: - - ```shell - helm upgrade nginx-ingress oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.19.0 --set controller.kind=deployment/daemonset --set controller.nginxplus=false/true --set controller.image.pullPolicy=Always --set serviceNameOverride="nginx-ingress-nginx-ingress" --set controller.name="" --set fullnameOverride="nginx-ingress-nginx-ingress" -f values.yaml - ``` - -6. Once the upgrade process has finished, use `kubectl describe` on the deployment to verify the change by -reviewing its events: - - ```shell - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal ScalingReplicaSet 9m11s deployment-controller Scaled up replica set nginx-ingress-nginx-ingress- to 1 - Normal ScalingReplicaSet 101s deployment-controller Scaled up replica set nginx-ingress-nginx-ingress- to 1 - Normal ScalingReplicaSet 98s deployment-controller Scaled down replica set nginx-ingress-nginx-ingress- to 0 from 1 - ``` - -{{%/tab%}} - -{{%tab name="Helm release name is not `nginx-ingress`"%}} - -1. Use `kubectl describe` on deployment/daemonset to get the `Selector` value: - - ```shell - kubectl describe deployment/daemonset -n - ``` - - Copy the key=value under ```Selector```, such as: - - ```shell - Selector: app=-nginx-ingress - ``` - -2. Checkout the latest available tag using `git checkout v3.3.2` - -3. Navigate to `/kubernates-ingress/deployments/helm-chart` - -4. Update the `selectorLabels: {}` field in the `values.yaml` file located at `/kubernates-ingress/deployments/helm-chart` -with the copied `Selector` value. - - ```shell - selectorLabels: {app: -nginx-ingress} - ``` - -5. Run `helm upgrade` with following arguments set: - - ```shell - --set serviceNameOverride="-nginx-ingress" - --set controller.name="" - ``` - - It could look as follows: - - ```shell - helm upgrade test-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.19.0 --set controller.kind=deployment/daemonset --set controller.nginxplus=false/true --set controller.image.pullPolicy=Always --set serviceNameOverride="test-release-nginx-ingress" --set controller.name="" -f values.yaml - ``` - -6. Once the upgrade process has finished, use `kubectl describe` on the deployment to verify the change by -reviewing its events: - - ```shell - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal ScalingReplicaSet 9m11s deployment-controller Scaled up replica set test-release-nginx-ingress- to 1 - Normal ScalingReplicaSet 101s deployment-controller Scaled up replica set test-release-nginx-ingress- to 1 - Normal ScalingReplicaSet 98s deployment-controller Scaled down replica set test-release-nginx-ingress- to 0 from 1 - ``` - -{{%/tab%}} - -{{}} - -## Running Multiple Ingress Controllers - -If you are running multiple Ingress Controller releases in your cluster with enabled custom resources, the releases will -share a single version of the CRDs. As a result, make sure that the Ingress Controller versions match the version of the -CRDs. Additionally, when uninstalling a release, ensure that you don’t remove the CRDs until there are no other Ingress -Controller releases running in the cluster. - -See [running multiple Ingress Controllers](https://docs.nginx.com/nginx-ingress-controller/installation/running-multiple-ingress-controllers/) -for more details. - -## Configuration - -The following tables lists the configurable parameters of the NGINX Ingress Controller chart and their default values. - -{{% table %}} -|Parameter | Description | Default | -| --- | --- | --- | -|`controller.name` | The name of the Ingress Controller daemonset or deployment. | Autogenerated | -|`controller.kind` | The kind of the Ingress Controller installation - deployment or daemonset. | deployment | -|`controller.annotations` | Allows for setting of `annotations` for deployment or daemonset. | {} | -|`controller.nginxplus` | Deploys the Ingress Controller for NGINX Plus. | false | -|`controller.nginxReloadTimeout` | The timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. | 60000 | -|`controller.hostNetwork` | Enables the Ingress Controller pods to use the host's network namespace. | false | -|`controller.dnsPolicy` | DNS policy for the Ingress Controller pods. | ClusterFirst | -|`controller.nginxDebug` | Enables debugging for NGINX. Uses the `nginx-debug` binary. Requires `error-log-level: debug` in the ConfigMap via `controller.config.entries`. | false | -|`controller.logLevel` | The log level of the Ingress Controller. | 1 | -|`controller.image.digest` | The image digest of the Ingress Controller. | None | -|`controller.image.repository` | The image repository of the Ingress Controller. | nginx/nginx-ingress | -|`controller.image.tag` | The tag of the Ingress Controller image. | 3.3.2 | -|`controller.image.pullPolicy` | The pull policy for the Ingress Controller image. | IfNotPresent | -|`controller.lifecycle` | The lifecycle of the Ingress Controller pods. | {} | -|`controller.customConfigMap` | The name of the custom ConfigMap used by the Ingress Controller. If set, then the default config is ignored. | "" | -|`controller.config.name` | The name of the ConfigMap used by the Ingress Controller. | Autogenerated | -|`controller.config.annotations` | The annotations of the Ingress Controller configmap. | {} | -|`controller.config.entries` | The entries of the ConfigMap for customizing NGINX configuration. See [ConfigMap resource docs](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) for the list of supported ConfigMap keys. | {} | -|`controller.customPorts` | A list of custom ports to expose on the NGINX Ingress Controller pod. Follows the conventional Kubernetes yaml syntax for container ports. | [] | -|`controller.defaultTLS.cert` | The base64-encoded TLS certificate for the default HTTPS server. **Note:** It is recommended that you specify your own certificate. Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. | -|`controller.defaultTLS.key` | The base64-encoded TLS key for the default HTTPS server. **Note:** It is recommended that you specify your own key. Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. | -|`controller.defaultTLS.secret` | The secret with a TLS certificate and key for the default HTTPS server. The value must follow the following format: `/`. Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. **Note:** Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. | None | -|`controller.wildcardTLS.cert` | The base64-encoded TLS certificate for every Ingress/VirtualServer host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection. | None | -|`controller.wildcardTLS.key` | The base64-encoded TLS key for every Ingress/VirtualServer host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection. | None | -|`controller.wildcardTLS.secret` | The secret with a TLS certificate and key for every Ingress/VirtualServer host that has TLS enabled but no secret specified. The value must follow the following format: `/`. Used as an alternative to specifying a certificate and key using `controller.wildcardTLS.cert` and `controller.wildcardTLS.key` parameters. | None | -|`controller.nodeSelector` | The node selector for pod assignment for the Ingress Controller pods. | {} | -|`controller.terminationGracePeriodSeconds` | The termination grace period of the Ingress Controller pod. | 30 | -|`controller.tolerations` | The tolerations of the Ingress Controller pods. | [] | -|`controller.affinity` | The affinity of the Ingress Controller pods. | {} | -|`controller.topologySpreadConstraints` | The topology spread constraints of the Ingress controller pods. | {} | -|`controller.env` | The additional environment variables to be set on the Ingress Controller pods. | [] | -|`controller.volumes` | The volumes of the Ingress Controller pods. | [] | -|`controller.volumeMounts` | The volumeMounts of the Ingress Controller pods. | [] | -|`controller.initContainers` | InitContainers for the Ingress Controller pods. | [] | -|`controller.extraContainers` | Extra (eg. sidecar) containers for the Ingress Controller pods. | [] | -|`controller.resources` | The resources of the Ingress Controller pods. | requests: cpu=100m,memory=128Mi | -|`controller.replicaCount` | The number of replicas of the Ingress Controller deployment. | 1 | -|`controller.ingressClass.name` | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of Kubernetes. | nginx | -|`controller.ingressClass.create` | Creates a new IngressClass object with the name `controller.ingressClass.name`. Set to `false` to use an existing ingressClass created using `kubectl` with the same name. If you use `helm upgrade`, do not change the values from the previous release as helm will delete IngressClass objects managed by helm. If you are upgrading from a release earlier than 3.3.2, do not set the value to false. | true | -|`controller.ingressClass.setAsDefaultIngress` | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass.name`. Requires `controller.ingressClass.create`. | false | -|`controller.watchNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchNamespace="default\,nginx-ingress"`. | "" | -|`controller.watchNamespaceLabel` | Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespace`. | "" | -|`controller.watchSecretNamespace` | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `controller.watchNamespace` and `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchSecretNamespace="default\,nginx-ingress"`. | "" | -|`controller.enableCustomResources` | Enable the custom resources. | true | -|`controller.enablePreviewPolicies` | Enable preview policies. This parameter is deprecated. To enable OIDC Policies please use `controller.enableOIDC` instead. | false | -|`controller.enableOIDC` | Enable OIDC policies. | false | -|`controller.enableTLSPassthrough` | Enable TLS Passthrough on default port 443. Requires `controller.enableCustomResources`. | false | -|`controller.tlsPassThroughPort` | Set the port for the TLS Passthrough. Requires `controller.enableCustomResources` and `controller.enableTLSPassthrough`. | 443 | -|`controller.enableCertManager` | Enable x509 automated certificate management for VirtualServer resources using cert-manager (cert-manager.io). Requires `controller.enableCustomResources`. | false | -|`controller.enableExternalDNS` | Enable integration with ExternalDNS for configuring public DNS entries for VirtualServer resources using [ExternalDNS](https://github.com/kubernetes-sigs/external-dns). Requires `controller.enableCustomResources`. | false | -|`controller.globalConfiguration.create` | Creates the GlobalConfiguration custom resource. Requires `controller.enableCustomResources`. | false | -|`controller.globalConfiguration.spec` | The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. | {} | -|`controller.enableSnippets` | Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. | false | -|`controller.healthStatus` | Add a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. Useful for external health-checking of the Ingress Controller. | false | -|`controller.healthStatusURI` | Sets the URI of health status location in the default server. Requires `controller.healthStatus`. | "/nginx-health" | -|`controller.nginxStatus.enable` | Enable the NGINX stub_status, or the NGINX Plus API. | true | -|`controller.nginxStatus.port` | Set the port where the NGINX stub_status or the NGINX Plus API is exposed. | 8080 | -|`controller.nginxStatus.allowCidrs` | Add IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas. | 127.0.0.1,::1 | -|`controller.priorityClassName` | The PriorityClass of the Ingress Controller pods. | None | -|`controller.service.create` | Creates a service to expose the Ingress Controller pods. | true | -|`controller.service.type` | The type of service to create for the Ingress Controller. | LoadBalancer | -|`controller.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | Local | -|`controller.service.annotations` | The annotations of the Ingress Controller service. | {} | -|`controller.service.extraLabels` | The extra labels of the service. | {} | -|`controller.service.loadBalancerIP` | The static IP address for the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | "" | -|`controller.service.externalIPs` | The list of external IPs for the Ingress Controller service. | [] | -|`controller.service.clusterIP` | The clusterIP for the Ingress Controller service, autoassigned if not specified. | "" | -|`controller.service.loadBalancerSourceRanges` | The IP ranges (CIDR) that are allowed to access the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | [] | -|`controller.service.name` | The name of the service. | Autogenerated | -|`controller.service.customPorts` | A list of custom ports to expose through the Ingress Controller service. Follows the conventional Kubernetes yaml syntax for service ports. | [] | -|`controller.service.httpPort.enable` | Enables the HTTP port for the Ingress Controller service. | true | -|`controller.service.httpPort.port` | The HTTP port of the Ingress Controller service. | 80 | -|`controller.service.httpPort.nodePort` | The custom NodePort for the HTTP port. Requires `controller.service.type` set to `NodePort`. | "" | -|`controller.service.httpPort.targetPort` | The target port of the HTTP port of the Ingress Controller service. | 80 | -|`controller.service.httpsPort.enable` | Enables the HTTPS port for the Ingress Controller service. | true | -|`controller.service.httpsPort.port` | The HTTPS port of the Ingress Controller service. | 443 | -|`controller.service.httpsPort.nodePort` | The custom NodePort for the HTTPS port. Requires `controller.service.type` set to `NodePort`. | "" | -|`controller.service.httpsPort.targetPort` | The target port of the HTTPS port of the Ingress Controller service. | 443 | -|`controller.serviceAccount.annotations` | The annotations of the Ingress Controller service account. | {} | -|`controller.serviceAccount.name` | The name of the service account of the Ingress Controller pods. Used for RBAC. | Autogenerated | -|`controller.serviceAccount.imagePullSecretName` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. | "" | -|`controller.serviceMonitor.name` | The name of the serviceMonitor. | Autogenerated | -|`controller.serviceMonitor.create` | Create a ServiceMonitor custom resource. | false | -|`controller.serviceMonitor.labels` | Kubernetes object labels to attach to the serviceMonitor object. | "" | -|`controller.serviceMonitor.selectorMatchLabels` | A set of labels to allow the selection of endpoints for the ServiceMonitor. | "" | -|`controller.serviceMonitor.endpoints` | A list of endpoints allowed as part of this ServiceMonitor. | "" | -|`controller.reportIngressStatus.enable` | Updates the address field in the status of Ingress resources with an external address of the Ingress Controller. You must also specify the source of the external address either through an external service via `controller.reportIngressStatus.externalService`, `controller.reportIngressStatus.ingressLink` or the `external-status-address` entry in the ConfigMap via `controller.config.entries`. **Note:** `controller.config.entries.external-status-address` takes precedence over the others. | true | -|`controller.reportIngressStatus.externalService` | Specifies the name of the service with the type LoadBalancer through which the Ingress Controller is exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. The default is autogenerated and enabled when `controller.service.create` is set to `true` and `controller.service.type` is set to `LoadBalancer`. | Autogenerated | -|`controller.reportIngressStatus.ingressLink` | Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. | "" | -|`controller.reportIngressStatus.enableLeaderElection` | Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress resources. `controller.reportIngressStatus.enable` must be set to `true`. | true | -|`controller.reportIngressStatus.leaderElectionLockName` | Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. controller.reportIngressStatus.enableLeaderElection must be set to true. | Autogenerated | -|`controller.reportIngressStatus.annotations` | The annotations of the leader election configmap. | {} | -|`controller.pod.annotations` | The annotations of the Ingress Controller pod. | {} | -|`controller.pod.extraLabels` | The additional extra labels of the Ingress Controller pod. | {} | -|`controller.appprotect.enable` | Enables the App Protect WAF module in the Ingress Controller. | false | -|`controller.appprotectdos.enable` | Enables the App Protect DoS module in the Ingress Controller. | false | -|`controller.appprotectdos.debug` | Enable debugging for App Protect DoS. | false | -|`controller.appprotectdos.maxDaemons` | Max number of ADMD instances. | 1 | -|`controller.appprotectdos.maxWorkers` | Max number of nginx processes to support. | Number of CPU cores in the machine | -|`controller.appprotectdos.memory` | RAM memory size to consume in MB. | 50% of free RAM in the container or 80MB, the smaller | -|`controller.readyStatus.enable` | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | true | -|`controller.readyStatus.port` | The HTTP port for the readiness endpoint. | 8081 | -|`controller.readyStatus.initialDelaySeconds` | The number of seconds after the Ingress Controller pod has started before readiness probes are initiated. | 0 | -|`controller.enableLatencyMetrics` | Enable collection of latency metrics for upstreams. Requires `prometheus.create`. | false | -|`controller.minReadySeconds` | Specifies the minimum number of seconds for which a newly created Pod should be ready without any of its containers crashing, for it to be considered available. [docs](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds) | 0 | -|`controller.autoscaling.enabled` | Enables HorizontalPodAutoscaling. | false | -|`controller.autoscaling.annotations` | The annotations of the Ingress Controller HorizontalPodAutoscaler. | {} | -|`controller.autoscaling.minReplicas` | Minimum number of replicas for the HPA. | 1 | -|`controller.autoscaling.maxReplicas` | Maximum number of replicas for the HPA. | 3 | -|`controller.autoscaling.targetCPUUtilizationPercentage` | The target CPU utilization percentage. | 50 | -|`controller.autoscaling.targetMemoryUtilizationPercentage` | The target memory utilization percentage. | 50 | -|`controller.podDisruptionBudget.enabled` | Enables PodDisruptionBudget. | false | -|`controller.podDisruptionBudget.annotations` | The annotations of the Ingress Controller pod disruption budget | {} | -|`controller.podDisruptionBudget.minAvailable` | The number of Ingress Controller pods that should be available. This is a mutually exclusive setting with "maxUnavailable". | 0 | -|`controller.podDisruptionBudget.maxUnavailable` | The number of Ingress Controller pods that can be unavailable. This is a mutually exclusive setting with "minAvailable". | 0 | -|`controller.strategy` | Specifies the strategy used to replace old Pods with new ones. Docs for [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) and [Daemonset update strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) | {} | -|`controller.disableIPV6` | Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. | false | -|`controller.readOnlyRootFilesystem` | Configure root filesystem as read-only and add volumes for temporary data. | false | -|`rbac.create` | Configures RBAC. | true | -|`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true | -|`prometheus.port` | Configures the port to scrape the metrics. | 9113 | -|`prometheus.scheme` | Configures the HTTP scheme to use for connections to the Prometheus endpoint. | http | -|`prometheus.secret` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Prometheus endpoint with TLS connections. | "" | -|`prometheus.service.create` | Create a Headless service to expose prometheus metrics. Requires `prometheus.create`. | false | -|`prometheus.service.labels` | Kubernetes object labels to attach to the service object. | {service: "nginx-ingress-prometheus-service"} | -|`prometheus.serviceMonitor.create` | Create a ServiceMonitor custom resource. Requires ServiceMonitor CRD to be installed. For the latest CRD, check the latest release on the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) GitHub repo under `example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml` | false | -|`prometheus.serviceMonitor.labels` | Kubernetes object labels to attach to the serviceMonitor object. | {} | -|`prometheus.serviceMonitor.selectorMatchLabels` | A set of labels to allow the selection of endpoints for the ServiceMonitor. | {service: "nginx-ingress-prometheus-service"} | -|`prometheus.serviceMonitor.endpoints` | A list of endpoints allowed as part of this ServiceMonitor. | [port: prometheus] | -|`serviceInsight.create` | Expose NGINX Plus Service Insight endpoint. | false | -|`serviceInsight.port` | Configures the port to expose endpoints. | 9114 | -|`serviceInsight.scheme` | Configures the HTTP scheme to use for connections to the Service Insight endpoint. | http | -|`serviceInsight.secret` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Service Insight endpoint with TLS connections. | "" | -|`serviceNameOverride` | Used to prevent cloud load balancers from being replaced due to service name change during helm upgrades. | "" | -|`nginxServiceMesh.enable` | Enable integration with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. Requires `controller.nginxplus`. | false | -|`nginxServiceMesh.enableEgress` | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false | -{{% /table %}} - -## Notes - -- The values-icp.yaml file is used for deploying the Ingress Controller on IBM Cloud Private. See the [blog - post](https://www.nginx.com/blog/nginx-ingress-controller-ibm-cloud-private/) for more details. -- The values-nsm.yaml file is used for deploying the Ingress Controller with NGINX Service Mesh. See the NGINX Service - Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. diff --git a/docs/content/installation/installation-with-manifests.md b/docs/content/installation/installation-with-manifests.md deleted file mode 100644 index c9554de4793..00000000000 --- a/docs/content/installation/installation-with-manifests.md +++ /dev/null @@ -1,312 +0,0 @@ ---- -title: Installation with Manifests -description: "This document describes how to install the NGINX Ingress Controller in your Kubernetes cluster using Kubernetes manifests." -weight: 1800 -doctypes: [""] -aliases: - - /installation/ -toc: true -docs: "DOCS-603" ---- - -## Prerequisites - -{{}} All documentation should only be used with the latest stable release, indicated on [the releases page](https://github.com/nginxinc/kubernetes-ingress/releases) of the GitHub repository. {{}} - -1. Make sure you have access to an NGINX Ingress Controller image: - - For NGINX Ingress Controller, use the image `nginx/nginx-ingress` from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress). - - For NGINX Plus Ingress Controller, see [here](/nginx-ingress-controller/installation/pulling-ingress-controller-image) for details on pulling the image from the F5 Docker registry. - - To pull from the F5 Container registry in your Kubernetes cluster, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here](/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret). - - You can also build your own image and push it to your private Docker registry by following the instructions from [here](/nginx-ingress-controller/installation/building-ingress-controller-image). -2. Clone the NGINX Ingress Controller repository and change into the deployments folder: - - ```shell - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 - cd kubernetes-ingress/deployments - ``` - - {{}}The above command will clone the branch of the latest NGINX Ingress Controller release, and all documentation assumes you are using it.{{}} - ---- - -## 1. Configure RBAC - -1. Create a namespace and a service account for NGINX Ingress Controller: - - ```shell - kubectl apply -f common/ns-and-sa.yaml - ``` - -2. Create a cluster role and cluster role binding for the service account: - - ```shell - kubectl apply -f rbac/rbac.yaml - ``` - -3. (App Protect only) Create the App Protect role and role binding: - - ```shell - kubectl apply -f rbac/ap-rbac.yaml - ``` - -4. (App Protect DoS only) Create the App Protect DoS role and role binding: - - ```shell - kubectl apply -f rbac/apdos-rbac.yaml - ``` - -{{}} To perform this step you must be a cluster admin. Follow the documentation of your Kubernetes platform to configure the admin access. For Google Kubernetes Engine, see their [Role-Based Access Control](https://cloud.google.com/kubernetes-engine/docs/how-to/role-based-access-control) documentation.{{}} - ---- - -## 2. Create Common Resources - -In this section, we create resources common for most of NGINX Ingress Controller installations: -{{}} -Installing the `default-server-secret.yaml` is optional and is required only if you are using the [default server TLS secret](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-default-server-tls-secret) command line argument. It is recommended that users provide their own certificate. -Otherwise, step 1 can be ignored. -{{}} - -1. Create a secret with a TLS certificate and a key for the default server in NGINX (below assumes you are in the `kubernetes-ingress/deployment` directory): - - ```console - kubectl apply -f ../examples/shared-examples/default-server-secret/default-server-secret.yaml - ``` - - {{}} The default server returns the Not Found page with the 404 status code for all requests for domains for which there are no Ingress rules defined. For testing purposes we include a self-signed certificate and key that we generated. However, we recommend that you use your own certificate and key. {{}} - -1. Create a config map for customizing NGINX configuration: - - ```console - kubectl apply -f common/nginx-config.yaml - ``` - -1. Create an IngressClass resource: - - ```console - kubectl apply -f common/ingress-class.yaml - ``` - - If you would like to set this NGINX Ingress Controller instance as the default, uncomment the annotation `ingressclass.kubernetes.io/is-default-class`. With this annotation set to true all the new Ingresses without an ingressClassName field specified will be assigned this IngressClass. - - {{}} NGINX Ingress Controller will fail to start without an IngressClass resource. {{}} - ---- - -## 3. Create Custom Resources - -{{}} -By default, it is required to create custom resource definitions for VirtualServer, VirtualServerRoute, TransportServer and Policy. Otherwise, NGINX Ingress Controller pods will not become `Ready`. If you'd like to disable that requirement, configure [`-enable-custom-resources`](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-global-configuration) command-line argument to `false` and skip this section. -{{}} - -1. Create custom resource definitions for [VirtualServer and VirtualServerRoute](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources), [TransportServer](/nginx-ingress-controller/configuration/transportserver-resource) and [Policy](/nginx-ingress-controller/configuration/policy-resource) resources: - - ```console - kubectl apply -f common/crds/k8s.nginx.org_virtualservers.yaml - kubectl apply -f common/crds/k8s.nginx.org_virtualserverroutes.yaml - kubectl apply -f common/crds/k8s.nginx.org_transportservers.yaml - kubectl apply -f common/crds/k8s.nginx.org_policies.yaml - ``` - -2. If you would like to use the TCP and UDP load balancing features, create a custom resource definition for the [GlobalConfiguration](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource) resource: - - ```console - kubectl apply -f common/crds/k8s.nginx.org_globalconfigurations.yaml - ``` - -3. If you would like to use the App Protect WAF module, you will need to create custom resource definitions for `APPolicy`, `APLogConf` and `APUserSig`: - - ```console - kubectl apply -f common/crds/appprotect.f5.com_aplogconfs.yaml - kubectl apply -f common/crds/appprotect.f5.com_appolicies.yaml - kubectl apply -f common/crds/appprotect.f5.com_apusersigs.yaml - ``` - -4. If you would like to use the App Protect DoS module, you will need to create custom resource definitions for `APDosPolicy`, `APDosLogConf` and `DosProtectedResource`: - - ```console - kubectl apply -f common/crds/appprotectdos.f5.com_apdoslogconfs.yaml - kubectl apply -f common/crds/appprotectdos.f5.com_apdospolicy.yaml - kubectl apply -f common/crds/appprotectdos.f5.com_dosprotectedresources.yaml - ``` - ---- - -## 4. Deploying NGINX Ingress Controller - -There are two options for deploying NGINX Ingress Controller: - -- *Deployment*. Use a Deployment if you plan to dynamically change the number of Ingress Controller replicas. -- *DaemonSet*. Use a DaemonSet for deploying the Ingress Controller on every node or a subset of nodes. - -Additionally, if you would like to use the NGINX App Protect DoS module, you'll need to deploy the Arbitrator. - -{{}} Before creating a Deployment or Daemonset resource, make sure to update the [command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments) of NGINX Ingress Controller container in the corresponding manifest file according to your requirements. {{}} - ---- - -### Deploying Arbitrator for NGINX App Protect DoS - -There are two steps for deploying NGINX Ingress Controller with the NGINX App Protect DoS module: - -1. Build your own image and push it to your private Docker registry by following the instructions from [here](/nginx-ingress-controller/app-protect-dos/installation#Build-the-app-protect-dos-arb-Docker-Image). - -1. Run the Arbitrator by using a Deployment and Service - - ```console - kubectl apply -f deployment/appprotect-dos-arb.yaml - kubectl apply -f service/appprotect-dos-arb-svc.yaml - ``` - ---- - -### 4.1 Running NGINX Ingress Controller - -#### Using a Deployment - -When you run NGINX Ingress Controller by using a Deployment, by default, Kubernetes will create one NGINX Ingress Controller pod. - -For NGINX, run: - -```console -kubectl apply -f deployment/nginx-ingress.yaml -``` - -For NGINX Plus, run: - -```console -kubectl apply -f deployment/nginx-plus-ingress.yaml -``` - -{{}} Update the `nginx-plus-ingress.yaml` with the chosen image from the F5 Container registry; or the container image that you have built. {{}} - ---- - -#### Using a DaemonSet - -When you run the Ingress Controller by using a DaemonSet, Kubernetes will create an Ingress Controller pod on every node of the cluster. - -{{}} Read the Kubernetes [DaemonSet docs](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) to learn how to run NGINX Ingress Controller on a subset of nodes instead of on every node of the cluster.{{}} - -For NGINX, run: - -```console -kubectl apply -f daemon-set/nginx-ingress.yaml -``` - -For NGINX Plus, run: - -```console -kubectl apply -f daemon-set/nginx-plus-ingress.yaml -``` - -{{}}Update `nginx-plus-ingress.yaml` with the chosen image from the F5 Container registry; or the container image that you have built.{{}} - ---- - -### 4.2 Check that NGINX Ingress Controller is Running - -Run the following command to make sure that the NGINX Ingress Controller pods are running: - -```console -kubectl get pods --namespace=nginx-ingress -``` - -## 5. Getting Access to NGINX Ingress Controller - -**If you created a daemonset**, ports 80 and 443 of NGINX Ingress Controller container are mapped to the same ports of the node where the container is running. To access NGINX Ingress Controller, use those ports and an IP address of any node of the cluster where the Ingress Controller is running. - -**If you created a deployment**, there are two options for accessing NGINX Ingress Controller pods: - -### 5.1 Create a Service for the NGINX Ingress Controller Pods - -#### Using a NodePort Service - -Create a service with the type *NodePort*: - -```console -kubectl create -f service/nodeport.yaml -``` - -Kubernetes will randomly allocate two ports on every node of the cluster. To access the Ingress Controller, use an IP address of any node of the cluster along with the two allocated ports. - -{{}} Read more about the type NodePort in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). {{}} - -#### Using a LoadBalancer Service - -1. Create a service using a manifest for your cloud provider: - - For GCP or Azure, run: - - ```shell - kubectl apply -f service/loadbalancer.yaml - ``` - - - For AWS, run: - - ```shell - kubectl apply -f service/loadbalancer-aws-elb.yaml - ``` - - Kubernetes will allocate a Classic Load Balancer (ELB) in TCP mode with the PROXY protocol enabled to pass the client's information (the IP address and the port). NGINX must be configured to use the PROXY protocol: - - Add the following keys to the config map file `nginx-config.yaml` from the Step 2: - - ```yaml - proxy-protocol: "True" - real-ip-header: "proxy_protocol" - set-real-ip-from: "0.0.0.0/0" - ``` - - - Update the config map: - - ```shell - kubectl apply -f common/nginx-config.yaml - ``` - - {{}} For AWS, additional options regarding an allocated load balancer are available, such as its type and SSL termination. Read the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer) to learn more. {{}} - - Kubernetes will allocate and configure a cloud load balancer for load balancing the Ingress Controller pods. -2. Use the public IP of the load balancer to access NGINX Ingress Controller. To get the public IP: - - For GCP or Azure, run: - - ```shell - kubectl get svc nginx-ingress --namespace=nginx-ingress - ``` - - - In case of AWS ELB, the public IP is not reported by `kubectl`, because the ELB IP addresses are not static. In general, you should rely on the ELB DNS name instead of the ELB IP addresses. However, for testing purposes, you can get the DNS name of the ELB using `kubectl describe` and then run `nslookup` to find the associated IP address: - - ```shell - kubectl describe svc nginx-ingress --namespace=nginx-ingress - ``` - - You can resolve the DNS name into an IP address using `nslookup`: - - ```shell - nslookup - ``` - - The public IP can be reported in the status of an ingress resource. See the [Reporting Resources Status doc](/nginx-ingress-controller/configuration/global-configuration/reporting-resources-status) for more details. - -{{}} Learn more about type LoadBalancer in the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer). {{}} - -## Uninstall NGINX Ingress Controller - -1. Delete the `nginx-ingress` namespace to uninstall NGINX Ingress Controller along with all the auxiliary resources that were created: - - ```shell - kubectl delete namespace nginx-ingress - ``` - -1. Delete the ClusterRole and ClusterRoleBinding: - - ```shell - kubectl delete clusterrole nginx-ingress - kubectl delete clusterrolebinding nginx-ingress - ``` - -1. Delete the Custom Resource Definitions: - - {{}} This step will also remove all associated Custom Resources. {{}} - - ```shell - kubectl delete -f common/crds/ - ``` diff --git a/docs/content/installation/installation-with-operator.md b/docs/content/installation/installation-with-operator.md deleted file mode 100644 index af6f476b70b..00000000000 --- a/docs/content/installation/installation-with-operator.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -title: Installation with the NGINX Ingress Operator - -description: "This document describes how to install the NGINX Ingress Controller in your Kubernetes cluster using the NGINX Ingress Operator." -weight: 2000 -doctypes: [""] -toc: true -docs: "DOCS-604" ---- - -This document describes how to install the F5 NGINX Ingress Controller in your Kubernetes cluster using the NGINX Ingress Operator. - -## Prerequisites - -**Note** All documentation should only be used with the latest stable release, indicated on [the releases page](https://github.com/nginxinc/kubernetes-ingress/releases) of the GitHub repository. - -1. Make sure you have access to the Ingress Controller image: - - For NGINX Ingress Controller, use the image `nginx/nginx-ingress` from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress). - - For NGINX Plus Ingress Controller, see [here](/nginx-ingress-controller/installation/pulling-ingress-controller-image) for details on how to pull the image from the F5 Docker registry. - - To pull from the F5 Container registry, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here](/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret). - - It is also possible to build your own image and push it to your private Docker registry by following the instructions from [here](/nginx-ingress-controller/installation/building-ingress-controller-image). -2. Install the NGINX Ingress Operator following the [instructions](https://github.com/nginxinc/nginx-ingress-helm-operator/blob/v2.0.2/docs/installation.md). -3. Create the SecurityContextConstraint as outlined in the ["Getting Started" instructions](https://github.com/nginxinc/nginx-ingress-helm-operator/blob/v2.0.2/README.md#getting-started). - -## 1. Create the NginxIngressController manifest - -Create a manifest `nginx-ingress-controller.yaml` with the following content: - -```yaml -apiVersion: charts.nginx.org/v1alpha1 -kind: NginxIngress -metadata: - name: nginxingress-sample - namespace: nginx-ingress -spec: - controller: - image: - pullPolicy: IfNotPresent - repository: nginx/nginx-ingress - tag: 3.3.2-ubi - ingressClass: - name: nginx - kind: deployment - nginxplus: false - replicaCount: 1 - serviceAccount: - imagePullSecretName: "" -``` - -**Note:** For NGINX Plus, change the `image.repository` and `image.tag` values and change `nginxPlus` to `True`. -If required, set the `serviceAccount.imagePullSecretName` to the name of the pre-created docker config secret that -should be associated with the ServiceAccount. - -## 2. Create the NginxIngressController - -```console -kubectl apply -f nginx-ingress-controller.yaml -``` - -A new instance of the NGINX Ingress Controller will be deployed by the NGINX Ingress Operator in the `default` -namespace with default parameters. - -To configure other parameters of the NginxIngressController resource, check the [documentation](https://github.com/nginxinc/nginx-ingress-helm-operator/blob/v2.0.2/docs/nginx-ingress-controller.md). - -**Note:** Some users reported an `OOMkilled` error when they deployed the NGINX Ingress Operator in a large cluster with -multiple namespaces and Kubernetes objects. This is due to the helm operator caching every Kubernetes object in the cluster, -and thus consuming too much system memory. If you encounter this issue, consider setting the operator to only watch one namespace. -If watching multiple namespaces is required in your use case, try manually increasing the memory limit for the operator. -Note that the value might be overwritten after a release update. We are working with the OpenShift team to resolve this issue. diff --git a/docs/content/installation/installing-nic/_index.md b/docs/content/installation/installing-nic/_index.md new file mode 100644 index 00000000000..51f8c4eacf7 --- /dev/null +++ b/docs/content/installation/installing-nic/_index.md @@ -0,0 +1,5 @@ +--- +title: Installing NGINX Ingress Controller +description: +weight: 100 +--- diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md new file mode 100644 index 00000000000..42f12b572dc --- /dev/null +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -0,0 +1,436 @@ +--- +title: Installation with Helm +description: This document explains how to install NGINX Ingress Controller in your Kubernetes cluster using Helm. +weight: 200 +doctypes: [""] +toc: true +docs: "DOCS-602" +--- +{{}} + + + +## Before you start + +{{}} All documentation should only be used with the latest stable release, indicated on [the releases page]({{< relref "releases.md" >}}) of the GitHub repository. {{}} + +- A [Kubernetes Version Supported by the Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/technical-specifications/#supported-kubernetes-versions) +- Helm 3.0+. +- If you’d like to use NGINX Plus: + - To pull from the F5 Container registry, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here](https://docs.nginx.com/nginx-ingress-controller/installation/nic-images/using-the-jwt-token-docker-secret). Make sure to specify the secret using `controller.serviceAccount.imagePullSecretName` parameter. + - Alternatively, pull an Ingress Controller image with NGINX Plus and push it to your private registry by following the instructions from [here]({{< relref "installation/nic-images/pulling-ingress-controller-image" >}}). + - Alternatively, you can build an Ingress Controller image with NGINX Plus and push it to your private registry by following the instructions from [here]({{< relref "installation/building-nginx-ingress-controller.md" >}}). + - Update the `controller.image.repository` field of the `values-plus.yaml` accordingly. +- If you’d like to use App Protect DoS, please install App Protect DoS Arbitrator [helm chart](https://github.com/nginxinc/nap-dos-arbitrator-helm-chart). Make sure to install in the same namespace as the NGINX Ingress Controller. Note that if you install multiple NGINX Ingress Controllers in the same namespace, they will need to share the same Arbitrator because it is not possible to install more than one Arbitrator in a single namespace. + +## CRDs + +By default, the Ingress Controller requires a number of custom resource definitions (CRDs) installed in the cluster. The Helm client will install those CRDs. If the CRDs are not installed, the Ingress Controller pods will not become `Ready`. + +If you do not use the custom resources that require those CRDs (which corresponds to `controller.enableCustomResources` set to `false` and `controller.appprotect.enable` set to `false` and `controller.appprotectdos.enable` set to `false`), the installation of the CRDs can be skipped by specifying `--skip-crds` for the helm install command. + +### Upgrading the CRDs + +To upgrade the CRDs, pull the chart sources as described in [Pulling the Chart](#pulling-the-chart) and then run: + +```shell +kubectl apply -f crds/ +``` + +{{}}The following warning is expected and can be ignored: `Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply`. + +Make sure to check the [release notes](https://www.github.com/nginxinc/kubernetes-ingress/releases) for a new release for any special upgrade procedures. +{{}} + +### Uninstalling the CRDs + +To remove the CRDs, pull the chart sources as described in [Pulling the Chart](#pulling-the-chart) and then run: + +```shell +kubectl delete -f crds/ +``` + +{{}}This command will delete all the corresponding custom resources in your cluster across all namespaces. Please ensure there are no custom resources that you want to keep and there are no other Ingress Controller releases running in the cluster.{{}} + +## Managing the Chart via OCI Registry + +### Installing the Chart + +To install the chart with the release name my-release (my-release is the name that you choose): + +- For NGINX: + + ```shell + helm install my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.18.1 + ``` + +- For NGINX Plus: (assuming you have pushed the Ingress Controller image `nginx-plus-ingress` to your private registry `myregistry.example.com`) + + ```shell + helm install my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.18.1 --set controller.image.repository=myregistry.example.com/nginx-plus-ingress --set controller.nginxplus=true + ``` + +This will install the latest `edge` version of the Ingress Controller from GitHub Container Registry. If you prefer to use Docker Hub, you can replace `ghcr.io/nginxinc/charts/nginx-ingress` with `registry-1.docker.io/nginxcharts/nginx-ingress`. + +### Upgrading the Chart + +Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, see [Upgrading the CRDs](#upgrading-the-crds). + +To upgrade the release `my-release`: + +```shell +helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.18.1 +``` + +### Uninstalling the Chart + +To uninstall/delete the release `my-release`: + +```shell +helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the release and deletes the release. + +Uninstalling the release does not remove the CRDs. To remove the CRDs, see [Uninstalling the CRDs](#uninstalling-the-crds). + +### Edge Version + +To test the latest changes in NGINX Ingress Controller before a new release, you can install the `edge` version. This version is built from the `main` branch of the NGINX Ingress Controller repository. +You can install the `edge` version by specifying the `--version` flag with the value `0.0.0-edge`: + +```shell +helm install my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.0.0-edge +``` + +> **Warning** +> +> The `edge` version is not intended for production use. It is intended for testing and development purposes only. + +## Managing the Chart via Sources + +### Pulling the Chart + +This step is required if you're installing the chart using its sources. Additionally, the step is also required for managing the custom resource definitions (CRDs), which the Ingress Controller requires by default, or for upgrading/deleting the CRDs. + +1. Pull the chart sources: + + ```shell + helm pull oci://ghcr.io/nginxinc/charts/nginx-ingress --untar --version 0.18.1 + ``` + +2. Change your working directory to nginx-ingress: + + ```shell + cd nginx-ingress + ``` + +### Installing the Chart + +To install the chart with the release name my-release (my-release is the name that you choose): + +- For NGINX: + + ```shell + helm install my-release . + ``` + +- For NGINX Plus: + + ```shell + helm install my-release -f values-plus.yaml . + ``` + +The command deploys the Ingress Controller in your Kubernetes cluster in the default configuration. The configuration section lists the parameters that can be configured during installation. + +### Upgrading the Chart + +Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, see [Upgrading the CRDs](#upgrading-the-crds). + +To upgrade the release `my-release`: + +```shell +helm upgrade my-release . +``` + +### Uninstalling the Chart + +To uninstall/delete the release `my-release`: + +```shell +helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the release and deletes the release. + +Uninstalling the release does not remove the CRDs. To remove the CRDs, see [Uninstalling the CRDs](#uninstalling-the-crds). + + +## Upgrading without downtime + +### Background + +In NGINX Ingress Controller version 3.1.0, [changes were introduced](https://github.com/nginxinc/kubernetes-ingress/pull/3606) to Helm resource names, labels and annotations to fit with Helm best practices. +When using Helm to upgrade from a version prior to 3.1.0, certain resources like Deployment, DaemonSet and Service will be recreated due to the aforementioned changes, which will result in downtime. + +Although the advisory is to update all resources in accordance with new naming convention, to avoid the downtime please follow the steps listed in this page. + +### Upgrade Steps + +{{}} The following steps apply to both 2.x and 3.0.x releases.{{}} + +The steps you should follow depend on the Helm release name: + +{{}} + +{{%tab name="Helm release name is `nginx-ingress`"%}} + +1. Use `kubectl describe` on deployment/daemonset to get the `Selector` value: + + ```shell + kubectl describe deployments -n + ``` + + Copy the key=value under `Selector`, such as: + + ```shell + Selector: app=nginx-ingress-nginx-ingress + ``` + +2. Checkout the latest available tag using `git checkout v3.3.0` + +3. Navigate to `/kubernates-ingress/deployments/helm-chart` + +4. Update the `selectorLabels: {}` field in the `values.yaml` file located at `/kubernates-ingress/deployments/helm-chart` with the copied `Selector` value. + ```shell + selectorLabels: {app: nginx-ingress-nginx-ingress} + ``` + +5. Run `helm upgrade` with following arguments set: + ```shell + --set serviceNameOverride="nginx-ingress-nginx-ingress" + --set controller.name="" + --set fullnameOverride="nginx-ingress-nginx-ingress" + ``` + It could look as follows: + + ```shell + helm upgrade nginx-ingress oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.19.0 --set controller.kind=deployment/daemonset --set controller.nginxplus=false/true --set controller.image.pullPolicy=Always --set serviceNameOverride="nginx-ingress-nginx-ingress" --set controller.name="" --set fullnameOverride="nginx-ingress-nginx-ingress" -f values.yaml + ``` + +6. Once the upgrade process has finished, use `kubectl describe` on the deployment to verify the change by reviewing its events: + ```shell + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal ScalingReplicaSet 9m11s deployment-controller Scaled up replica set nginx-ingress-nginx-ingress- to 1 + Normal ScalingReplicaSet 101s deployment-controller Scaled up replica set nginx-ingress-nginx-ingress- to 1 + Normal ScalingReplicaSet 98s deployment-controller Scaled down replica set nginx-ingress-nginx-ingress- to 0 from 1 + ``` +{{%/tab%}} + +{{%tab name="Helm release name is not `nginx-ingress`"%}} + +1. Use `kubectl describe` on deployment/daemonset to get the `Selector` value: + + ```shell + kubectl describe deployment/daemonset -n + ``` + + Copy the key=value under ```Selector```, such as: + + ```shell + Selector: app=-nginx-ingress + ``` + +2. Checkout the latest available tag using `git checkout v3.3.0` + +3. Navigate to `/kubernates-ingress/deployments/helm-chart` + +4. Update the `selectorLabels: {}` field in the `values.yaml` file located at `/kubernates-ingress/deployments/helm-chart` with the copied `Selector` value. + + ```shell + selectorLabels: {app: -nginx-ingress} + ``` + +5. Run `helm upgrade` with following arguments set: + + ```shell + --set serviceNameOverride="-nginx-ingress" + --set controller.name="" + ``` + + It could look as follows: + + ```shell + helm upgrade test-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.19.0 --set controller.kind=deployment/daemonset --set controller.nginxplus=false/true --set controller.image.pullPolicy=Always --set serviceNameOverride="test-release-nginx-ingress" --set controller.name="" -f values.yaml + ``` + +6. Once the upgrade process has finished, use `kubectl describe` on the deployment to verify the change by reviewing its events: + + ```shell + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal ScalingReplicaSet 9m11s deployment-controller Scaled up replica set test-release-nginx-ingress- to 1 + Normal ScalingReplicaSet 101s deployment-controller Scaled up replica set test-release-nginx-ingress- to 1 + Normal ScalingReplicaSet 98s deployment-controller Scaled down replica set test-release-nginx-ingress- to 0 from 1 + ``` + +{{%/tab%}} + +{{}} + + +## Running Multiple Ingress Controllers + +If you are running multiple Ingress Controller releases in your cluster with enabled custom resources, the releases will share a single version of the CRDs. As a result, make sure that the Ingress Controller versions match the version of the CRDs. Additionally, when uninstalling a release, ensure that you don’t remove the CRDs until there are no other Ingress Controller releases running in the cluster. + +See [running multiple Ingress Controllers]({{< relref "installation/running-multiple-ingress-controllers.md" >}}) for more details. + +## Configuration + +The following tables lists the configurable parameters of the NGINX Ingress Controller chart and their default values. + +{{}} +|Parameter | Description | Default | +| --- | --- | --- | +| **controller.name** | The name of the Ingress Controller daemonset or deployment. | Autogenerated | +| **controller.kind** | The kind of the Ingress Controller installation - deployment or daemonset. | deployment | +| **controller.annotations** | Allows for setting of `annotations` for deployment or daemonset. | {} | +| **controller.nginxplus** | Deploys the Ingress Controller for NGINX Plus. | false | +| **controller.nginxReloadTimeout** | The timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. | 60000 | +| **controller.hostNetwork** | Enables the Ingress Controller pods to use the host's network namespace. | false | +| **controller.dnsPolicy** | DNS policy for the Ingress Controller pods. | ClusterFirst | +| **controller.nginxDebug** | Enables debugging for NGINX. Uses the `nginx-debug` binary. Requires `error-log-level: debug` in the ConfigMap via `controller.config.entries`. | false | +| **controller.logLevel** | The log level of the Ingress Controller. | 1 | +| **controller.image.digest** | The image digest of the Ingress Controller. | None | +| **controller.image.repository** | The image repository of the Ingress Controller. | nginx/nginx-ingress | +| **controller.image.tag** | The tag of the Ingress Controller image. | 3.2.1 | +| **controller.image.pullPolicy** | The pull policy for the Ingress Controller image. | IfNotPresent | +| **controller.lifecycle** | The lifecycle of the Ingress Controller pods. | {} | +| **controller.customConfigMap** | The name of the custom ConfigMap used by the Ingress Controller. If set, then the default config is ignored. | "" | +| **controller.config.name** | The name of the ConfigMap used by the Ingress Controller. | Autogenerated | +| **controller.config.annotations** | The annotations of the Ingress Controller configmap. | {} | +| **controller.config.entries** | The entries of the ConfigMap for customizing NGINX configuration. See [ConfigMap resource docs](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) for the list of supported ConfigMap keys. | {} | +| **controller.customPorts** | A list of custom ports to expose on the NGINX Ingress Controller pod. Follows the conventional Kubernetes yaml syntax for container ports. | [] | +| **controller.defaultTLS.cert** | The base64-encoded TLS certificate for the default HTTPS server. **Note:** It is recommended that you specify your own certificate. Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. | +| **controller.defaultTLS.key** | The base64-encoded TLS key for the default HTTPS server. **Note:** It is recommended that you specify your own key. Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. | +| **controller.defaultTLS.secret** | The secret with a TLS certificate and key for the default HTTPS server. The value must follow the following format: `/`. Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. **Note:** Alternatively, omitting the default server secret completely will configure NGINX to reject TLS connections to the default server. | None | +| **controller.wildcardTLS.cert** | The base64-encoded TLS certificate for every Ingress/VirtualServer host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection. | None | +| **controller.wildcardTLS.key** | The base64-encoded TLS key for every Ingress/VirtualServer host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress/VirtualServer hosts NGINX will break any attempt to establish a TLS connection. | None | +| **controller.wildcardTLS.secret** | The secret with a TLS certificate and key for every Ingress/VirtualServer host that has TLS enabled but no secret specified. The value must follow the following format: `/`. Used as an alternative to specifying a certificate and key using `controller.wildcardTLS.cert` and `controller.wildcardTLS.key` parameters. | None | +| **controller.nodeSelector** | The node selector for pod assignment for the Ingress Controller pods. | {} | +| **controller.terminationGracePeriodSeconds** | The termination grace period of the Ingress Controller pod. | 30 | +| **controller.tolerations** | The tolerations of the Ingress Controller pods. | [] | +| **controller.affinity** | The affinity of the Ingress Controller pods. | {} | +| **controller.topologySpreadConstraints** | The topology spread constraints of the Ingress controller pods. | {} | +| **controller.env** | The additional environment variables to be set on the Ingress Controller pods. | [] | +| **controller.volumes** | The volumes of the Ingress Controller pods. | [] | +| **controller.volumeMounts** | The volumeMounts of the Ingress Controller pods. | [] | +| **controller.initContainers** | InitContainers for the Ingress Controller pods. | [] | +| **controller.extraContainers** | Extra (eg. sidecar) containers for the Ingress Controller pods. | [] | +| **controller.resources** | The resources of the Ingress Controller pods. | requests: cpu=100m,memory=128Mi | +| **controller.replicaCount** | The number of replicas of the Ingress Controller deployment. | 1 | +| **controller.ingressClass** | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of Kubernetes. | nginx | +| **controller.setAsDefaultIngress** | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. | false | +| **controller.watchNamespace** | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchNamespace="default\,nginx-ingress"`. | "" | +| **controller.watchNamespaceLabel** | Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespace`. | "" | +| **controller.watchSecretNamespace** | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `controller.watchNamespace` and `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchSecretNamespace="default\,nginx-ingress"`. | "" | +| **controller.enableCustomResources** | Enable the custom resources. | true | +| **controller.enablePreviewPolicies** | Enable preview policies. This parameter is deprecated. To enable OIDC Policies please use `controller.enableOIDC` instead. | false | +| **controller.enableOIDC** | Enable OIDC policies. | false | +| **controller.enableTLSPassthrough** | Enable TLS Passthrough on default port 443. Requires `controller.enableCustomResources`. | false | +| **controller.tlsPassThroughPort** | Set the port for the TLS Passthrough. Requires `controller.enableCustomResources` and `controller.enableTLSPassthrough`. | 443 | +| **controller.enableCertManager** | Enable x509 automated certificate management for VirtualServer resources using cert-manager (cert-manager.io). Requires `controller.enableCustomResources`. | false | +| **controller.enableExternalDNS** | Enable integration with ExternalDNS for configuring public DNS entries for VirtualServer resources using [ExternalDNS](https://github.com/kubernetes-sigs/external-dns). Requires `controller.enableCustomResources`. | false | +| **controller.globalConfiguration.create** | Creates the GlobalConfiguration custom resource. Requires `controller.enableCustomResources`. | false | +| **controller.globalConfiguration.spec** | The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. | {} | +| **controller.enableSnippets** | Enable custom NGINX configuration snippets in Ingress, VirtualServer, VirtualServerRoute and TransportServer resources. | false | +| **controller.healthStatus** | Add a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. Useful for external health-checking of the Ingress Controller. | false | +| **controller.healthStatusURI** | Sets the URI of health status location in the default server. Requires `controller.healthStatus`. | "/nginx-health" | +| **controller.nginxStatus.enable** | Enable the NGINX stub_status, or the NGINX Plus API. | true | +| **controller.nginxStatus.port** | Set the port where the NGINX stub_status or the NGINX Plus API is exposed. | 8080 | +| **controller.nginxStatus.allowCidrs** | Add IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas. | 127.0.0.1,::1 | +| **controller.priorityClassName** | The PriorityClass of the Ingress Controller pods. | None | +| **controller.service.create** | Creates a service to expose the Ingress Controller pods. | true | +| **controller.service.type** | The type of service to create for the Ingress Controller. | LoadBalancer | +| **controller.service.externalTrafficPolicy** | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | Local | +| **controller.service.annotations** | The annotations of the Ingress Controller service. | {} | +| **controller.service.extraLabels** | The extra labels of the service. | {} | +| **controller.service.loadBalancerIP** | The static IP address for the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | "" | +| **controller.service.externalIPs** | The list of external IPs for the Ingress Controller service. | [] | +| **controller.service.loadBalancerSourceRanges** | The IP ranges (CIDR) that are allowed to access the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | [] | +| **controller.service.name** | The name of the service. | Autogenerated | +| **controller.service.customPorts** | A list of custom ports to expose through the Ingress Controller service. Follows the conventional Kubernetes yaml syntax for service ports. | [] | +| **controller.service.httpPort.enable** | Enables the HTTP port for the Ingress Controller service. | true | +| **controller.service.httpPort.port** | The HTTP port of the Ingress Controller service. | 80 | +| **controller.service.httpPort.nodePort** | The custom NodePort for the HTTP port. Requires `controller.service.type` set to `NodePort`. | "" | +| **controller.service.httpPort.targetPort** | The target port of the HTTP port of the Ingress Controller service. | 80 | +| **controller.service.httpsPort.enable** | Enables the HTTPS port for the Ingress Controller service. | true | +| **controller.service.httpsPort.port** | The HTTPS port of the Ingress Controller service. | 443 | +| **controller.service.httpsPort.nodePort** | The custom NodePort for the HTTPS port. Requires `controller.service.type` set to `NodePort`. | "" | +| **controller.service.httpsPort.targetPort** | The target port of the HTTPS port of the Ingress Controller service. | 443 | +| **controller.serviceAccount.annotations** | The annotations of the Ingress Controller service account. | {} | +| **controller.serviceAccount.name** | The name of the service account of the Ingress Controller pods. Used for RBAC. | Autogenerated | +| **controller.serviceAccount.imagePullSecretName** | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. | "" | +| **controller.serviceMonitor.name** | The name of the serviceMonitor. | Autogenerated | +| **controller.serviceMonitor.create** | Create a ServiceMonitor custom resource. | false | +| **controller.serviceMonitor.labels** | Kubernetes object labels to attach to the serviceMonitor object. | "" | +| **controller.serviceMonitor.selectorMatchLabels** | A set of labels to allow the selection of endpoints for the ServiceMonitor. | "" | +| **controller.serviceMonitor.endpoints** | A list of endpoints allowed as part of this ServiceMonitor. | "" | +| **controller.reportIngressStatus.enable** | Updates the address field in the status of Ingress resources with an external address of the Ingress Controller. You must also specify the source of the external address either through an external service via `controller.reportIngressStatus.externalService`, `controller.reportIngressStatus.ingressLink` or the `external-status-address` entry in the ConfigMap via `controller.config.entries`. **Note:** `controller.config.entries.external-status-address` takes precedence over the others. | true | +| **controller.reportIngressStatus.externalService** | Specifies the name of the service with the type LoadBalancer through which the Ingress Controller is exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. The default is autogenerated and enabled when `controller.service.create` is set to `true` and `controller.service.type` is set to `LoadBalancer`. | Autogenerated | +| **controller.reportIngressStatus.ingressLink** | Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. | "" | +| **controller.reportIngressStatus.enableLeaderElection** | Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress resources. `controller.reportIngressStatus.enable` must be set to `true`. | true | +| **controller.reportIngressStatus.leaderElectionLockName** | Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. controller.reportIngressStatus.enableLeaderElection must be set to true. | Autogenerated | +| **controller.reportIngressStatus.annotations** | The annotations of the leader election configmap. | {} | +| **controller.pod.annotations** | The annotations of the Ingress Controller pod. | {} | +| **controller.pod.extraLabels** | The additional extra labels of the Ingress Controller pod. | {} | +| **controller.appprotect.enable** | Enables the App Protect WAF module in the Ingress Controller. | false | +| **controller.appprotectdos.enable** | Enables the App Protect DoS module in the Ingress Controller. | false | +| **controller.appprotectdos.debug** | Enable debugging for App Protect DoS. | false | +| **controller.appprotectdos.maxDaemons** | Max number of ADMD instances. | 1 | +| **controller.appprotectdos.maxWorkers** | Max number of nginx processes to support. | Number of CPU cores in the machine | +| **controller.appprotectdos.memory** | RAM memory size to consume in MB. | 50% of free RAM in the container or 80MB, the smaller | +| **controller.readyStatus.enable** | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | true | +| **controller.readyStatus.port** | The HTTP port for the readiness endpoint. | 8081 | +| **controller.readyStatus.initialDelaySeconds** | The number of seconds after the Ingress Controller pod has started before readiness probes are initiated. | 0 | +| **controller.enableLatencyMetrics** | Enable collection of latency metrics for upstreams. Requires `prometheus.create`. | false | +| **controller.minReadySeconds** | Specifies the minimum number of seconds for which a newly created Pod should be ready without any of its containers crashing, for it to be considered available. [docs](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#min-ready-seconds) | 0 | +| **controller.autoscaling.enabled** | Enables HorizontalPodAutoscaling. | false | +| **controller.autoscaling.annotations** | The annotations of the Ingress Controller HorizontalPodAutoscaler. | {} | +| **controller.autoscaling.minReplicas** | Minimum number of replicas for the HPA. | 1 | +| **controller.autoscaling.maxReplicas** | Maximum number of replicas for the HPA. | 3 | +| **controller.autoscaling.targetCPUUtilizationPercentage** | The target CPU utilization percentage. | 50 | +| **controller.autoscaling.targetMemoryUtilizationPercentage** | The target memory utilization percentage. | 50 | +| **controller.podDisruptionBudget.enabled** | Enables PodDisruptionBudget. | false | +| **controller.podDisruptionBudget.annotations** | The annotations of the Ingress Controller pod disruption budget | {} | +| **controller.podDisruptionBudget.minAvailable** | The number of Ingress Controller pods that should be available. This is a mutually exclusive setting with "maxUnavailable". | 0 | +| **controller.podDisruptionBudget.maxUnavailable** | The number of Ingress Controller pods that can be unavailable. This is a mutually exclusive setting with "minAvailable". | 0 | +| **controller.strategy** | Specifies the strategy used to replace old Pods with new ones. Docs for [Deployment update strategy](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy) and [Daemonset update strategy](https://kubernetes.io/docs/tasks/manage-daemon/update-daemon-set/#daemonset-update-strategy) | {} | +| **controller.disableIPV6** | Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. | false | +| **controller.readOnlyRootFilesystem** | Configure root filesystem as read-only and add volumes for temporary data. | false | +| **rbac.create** | Configures RBAC. | true | +| **prometheus.create** | Expose NGINX or NGINX Plus metrics in the Prometheus format. | true | +| **prometheus.port** | Configures the port to scrape the metrics. | 9113 | +| **prometheus.scheme** | Configures the HTTP scheme to use for connections to the Prometheus endpoint. | http | +| **prometheus.secret** | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Prometheus endpoint with TLS connections. | "" | +| **serviceInsight.create** | Expose NGINX Plus Service Insight endpoint. | false | +| **serviceInsight.port** | Configures the port to expose endpoints. | 9114 | +| **serviceInsight.scheme** | Configures the HTTP scheme to use for connections to the Service Insight endpoint. | http | +| **serviceInsight.secret** | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Service Insight endpoint with TLS connections. | "" | +| **serviceNameOverride** | Used to prevent cloud load balancers from being replaced due to service name change during helm upgrades. | "" | +| **nginxServiceMesh.enable** | Enable integration with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. Requires `controller.nginxplus`. | false | +| **nginxServiceMesh.enableEgress** | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false | +{{}} + + + +## Notes + +- The values-icp.yaml file is used for deploying the Ingress Controller on IBM Cloud Private. See the [blog post](https://www.nginx.com/blog/nginx-ingress-controller-ibm-cloud-private/) for more details. +- The values-nsm.yaml file is used for deploying the Ingress Controller with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. diff --git a/docs/content/installation/installing-nic/installation-with-manifests.md b/docs/content/installation/installing-nic/installation-with-manifests.md new file mode 100644 index 00000000000..21d9507574c --- /dev/null +++ b/docs/content/installation/installing-nic/installation-with-manifests.md @@ -0,0 +1,228 @@ +--- +title: Installation with Manifests +description: "This guide explains how to install NGINX Ingress Controller in a Kubernetes cluster using manifests. In addition, it provides instructions on how to set up role-based access control, create both common and custom resources, and uninstall NGINX Ingress Controller." +weight: 100 +doctypes: [""] +aliases: + - /installation/ +toc: true +docs: "DOCS-603" +--- + +{{}} + +## Before you start + +### Get the NGINX Controller Image + +{{}}Always use the most up-to-date stable release listed on the [releases page]({{< relref "releases.md" >}}).{{}} + +Choose one of the following methods to get the NGINX Ingress Controller image: + +- **NGINX Ingress Controller**: Download the image `nginx/nginx-ingress` from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress). +- **NGINX Plus Ingress Controller**: You have two options for this, both requiring an NGINX Ingress Controller subscription. + + - Download the image using your NGINX Ingress Controller subscription certificate and key. See the [Getting the F5 Registry NGINX Ingress Controller Image]({{< relref "installation/nic-images/pulling-ingress-controller-image.md" >}}) guide. + - Use your NGINX Ingress Controller subscription JWT token to get the image: Instructions are in [Getting the NGINX Ingress Controller Image with JWT]({{< relref "installation/nic-images/using-the-jwt-token-docker-secret.md" >}}). + +- **Build your own image**: To build your own image, follow the [Building NGINX Ingress Controller]({{< relref "installation/building-nginx-ingress-controller.md" >}}) guide. + +### Clone the repository + +Clone the NGINX Ingress Controller repository and go to the _deployments_ folder. Replace `` with the specific release you want to use. + +```shell +git clone https://github.com/nginxinc/kubernetes-ingress.git --branch +cd kubernetes-ingress/deployments +``` + +For example, if you want to use version 3.2.1, the command would be `git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1`. + +This guide assumes you are using the latest release. + +--- + +## Set up role-based access control (RBAC) {#configure-rbac} + +{{< include "rbac/set-up-rbac.md" >}} + +--- + +## Create common resources {#create-common-resources} + +{{< include "installation/create-common-resources.md" >}} + +--- + +## Create custom resources {#create-custom-resources} + +To make sure your NGINX Ingress Controller pods reach the `Ready` state, you'll need to create custom resource definitions (CRDs) for various components. Alternatively, you can disable this requirement by setting the `-enable-custom-resources` command-line argument to `false`. + +### Core custom resource definitions + +1. Create CRDs for [VirtualServer and VirtualServerRoute]({{< relref "configuration/virtualserver-and-virtualserverroute-resources.md" >}}), [TransportServer]({{< relref "configuration/transportserver-resource.md" >}}), and [Policy]({{< relref "configuration/policy-resource.md" >}}): + + ```shell + kubectl apply -f common/crds/k8s.nginx.org_virtualservers.yaml + kubectl apply -f common/crds/k8s.nginx.org_virtualserverroutes.yaml + kubectl apply -f common/crds/k8s.nginx.org_transportservers.yaml + kubectl apply -f common/crds/k8s.nginx.org_policies.yaml + ``` + +### Optional custom resource definitions + +1. (Optional) For TCP and UDP load balancing, create a cCRD for [GlobalConfiguration]({{< relref "configuration/global-configuration/globalconfiguration-resource.md" >}}): + + ```shell + kubectl apply -f common/crds/k8s.nginx.org_globalconfigurations.yaml + ``` + +2. (Optional) For the NGINX App Protect WAF module, create CRDs for `APPolicy`, `APLogConf` and `APUserSig`: + + ```shell + kubectl apply -f common/crds/appprotect.f5.com_aplogconfs.yaml + kubectl apply -f common/crds/appprotect.f5.com_appolicies.yaml + kubectl apply -f common/crds/appprotect.f5.com_apusersigs.yaml + ``` + +3. (Optional) For the NGINX App Protect DoS module, create CRDs for `APDosPolicy`, `APDosLogConf` and `DosProtectedResource`: + + ```shell + kubectl apply -f common/crds/appprotectdos.f5.com_apdoslogconfs.yaml + kubectl apply -f common/crds/appprotectdos.f5.com_apdospolicy.yaml + kubectl apply -f common/crds/appprotectdos.f5.com_dosprotectedresources.yaml + ``` + +--- + +## Deploy NGINX Ingress Controller {#deploy-ingress-controller} + +You have two options for deploying NGINX Ingress Controller: + +- **Deployment**. Choose this method for the flexibility to dynamically change the number of NGINX Ingress Controller replicas. +- **DaemonSet**. Choose this method if you want NGINX Ingress Controller to run on all nodes or a subset of nodes. + +Before you start, update the [command-line arguments]({{< relref "configuration/global-configuration/command-line-arguments.md" >}}) for the NGINX Ingress Controller container in the relevant manifest file to meet your specific requirements. + +### Using a Deployment + +{{< include "installation/manifests/deployment.md" >}} + +### Using a DaemonSet + +{{< include "installation/manifests/daemonset.md" >}} + +--- + +## Confirm NGINX Ingress Controller is running + +{{< include "installation/manifests/verify-pods-are-running.md" >}} + +--- + +## How to access NGINX Ingress Controller + +### Using a Deployment + +For Deployments, you have two options for accessing NGINX Ingress Controller pods. + +#### Option 1: Create a NodePort service + +For more information about the _NodePort_ service, refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport). + +1. To create a service of type *NodePort*, run: + + ```shell + kubectl create -f service/nodeport.yaml + ``` + + Kubernetes automatically allocates two ports on every node in the cluster. You can access NGINX Ingress Controller by combining any node's IP address with these ports. + +#### Option 2: Create a LoadBalancer service + +For more information about the _LoadBalancer_ service, refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer). + +1. To set up a _LoadBalancer_ service, run one of the following commands based on your cloud provider: + + - GCP or Azure: + + ```shell + kubectl apply -f service/loadbalancer.yaml + ``` + + - AWS: + + ```shell + kubectl apply -f service/loadbalancer-aws-elb.yaml + ``` + + If yu're using AWS, Kubernetes will set up a Classic Load Balancer (ELB) in TCP mode. This load balancer will have the PROXY protocol enabled to pass along the client's IP address and port. + +2. AWS users: Follow these additional steps to work with ELB in TCP mode. + + - Add the following keys to the `nginx-config.yaml` ConfigMap file, which you created in the [Create common resources](#create-common-resources) section. + + ```yaml + proxy-protocol: "True" + real-ip-header: "proxy_protocol" + set-real-ip-from: "0.0.0.0/0" + ``` + + - Update the ConfigMap: + + ```shell + kubectl apply -f common/nginx-config.yaml + ``` + + {{}}AWS users have more customization options for their load balancers. These include choosing the load balancer type and configuring SSL termination. Refer to the [Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/service/#type-loadbalancer) to learn more. {{}} + +3. To access NGINX Ingress Controller, get the public IP of your load balancer. + + - For GCP or Azure, run: + + ```shell + kubectl get svc nginx-ingress --namespace=nginx-ingress + ``` + + - For AWS find the DNS name: + + ```shell + kubectl describe svc nginx-ingress --namespace=nginx-ingress + ``` + + Resolve the DNS name into an IP address using `nslookup`: + + ```shell + nslookup + ``` + + You can also find more details about the public IP in the status section of an ingress resource. For more details, refer to the [Reporting Resources Status doc]({{< relref "configuration/global-configuration/reporting-resources-status.md" >}}). + +### Using a DaemonSet + +Connect to ports 80 and 443 using the IP address of any node in the cluster where NGINX Ingress Controller is running. + +--- + +## Uninstall NGINX Ingress Controller + +{{}}Proceed with caution when performing these steps, as they will remove NGINX Ingress Controller and all related resources, potentially affecting your running services.{{}} + +1. **Delete the nginx-ingress namespace**: To remove NGINX Ingress Controller and all auxiliary resources, run: + + ```shell + kubectl delete namespace nginx-ingress + ``` + +2. **Remove the cluster role and cluster role binding**: + + ```shell + kubectl delete clusterrole nginx-ingress + kubectl delete clusterrolebinding nginx-ingress + ``` + +3. **Delete the Custom Resource Definitions**: Be aware that this step will also erase all associated custom resources. To proceed, run: + + ```shell + kubectl delete -f common/crds/ + ``` diff --git a/docs/content/installation/installing-nic/installation-with-operator.md b/docs/content/installation/installing-nic/installation-with-operator.md new file mode 100644 index 00000000000..70241d17313 --- /dev/null +++ b/docs/content/installation/installing-nic/installation-with-operator.md @@ -0,0 +1,80 @@ +--- +title: Installation with the NGINX Ingress Operator +description: "This document explains how to install NGINX Ingress Controller in your Kubernetes cluster using NGINX Ingress Operator." +weight: 300 +doctypes: [""] +toc: true +docs: "DOCS-604" +--- + +{{}} + + + +{{< note >}} +NGINX Ingress Operator isn't compatible with NGINX Ingress Controller 3.2.0 at this time. We'll update this guide and remove this note when we release a compatible version. +{{< /note >}} + +## Before you start + +{{}} We recommend the most recent stable version of NGINX Ingress Controller, available on the GitHub repository's [releases page]({{< relref "releases.md" >}}). {{}} + +1. Make sure you have access to the NGINX Ingress Controller image: + + - For NGINX Ingress Controller, use the image `nginx/nginx-ingress` from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress). + - For NGINX Plus Ingress Controller, see [here]({{< relref "installation/nic-images/pulling-ingress-controller-image" >}}) for details on how to pull the image from the F5 Docker registry. + - To pull from the F5 Container registry, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here]({{< relref "installation/nic-images/using-the-jwt-token-docker-secret" >}}). + - It is also possible to build your own image and push it to your private Docker registry by following the instructions from [here]({{< relref "installation/building-nginx-ingress-controller.md" >}})). + +2. Install the NGINX Ingress Operator following the [instructions](https://github.com/nginxinc/nginx-ingress-helm-operator/blob/v1.4.1/docs/installation.md). +3. Create the SecurityContextConstraint as outlined in the ["Getting Started" instructions](https://github.com/nginxinc/nginx-ingress-helm-operator/blob/v1.4.1/README.md#getting-started). + +## Create the NGINX Ingress Controller manifest + +Create a manifest `nginx-ingress-controller.yaml` with the following content: + +```yaml +apiVersion: charts.nginx.org/v1alpha1 +kind: NginxIngress +metadata: + name: nginxingress-sample + namespace: nginx-ingress +spec: + controller: + image: + pullPolicy: IfNotPresent + repository: nginx/nginx-ingress + tag: 3.2.1-ubi + ingressClass: nginx + kind: deployment + nginxplus: false + replicaCount: 1 + serviceAccount: + imagePullSecretName: "" +``` + +{{}}For NGINX Plus, change the `image.repository` and `image.tag` values and change `nginxPlus` to `True`. If required, set the `serviceAccount.imagePullSecretName` to the name of the pre-created docker config secret that should be associated with the ServiceAccount.{{}} + +## Deploy NGINX Ingress Controller + +```shell +kubectl apply -f nginx-ingress-controller.yaml +``` + +A new instance of NGINX Ingress Controller will be deployed by the NGINX Ingress Operator in the `default` namespace with default parameters. + +To configure other parameters of the NginxIngressController resource, check the [documentation](https://github.com/nginxinc/nginx-ingress-helm-operator/blob/v1.4.1/docs/nginx-ingress-controller.md). + +## Troubleshooting + +If you experience an `OOMkilled` error when deploying the NGINX Ingress Operator in a large cluster, it's likely because the Helm operator is caching all Kubernetes objects and using up too much memory. If you encounter this issue, try the following solutions: + +- Set the operator to only watch one namespace. +- If monitoring multiple namespaces is required, consider manually increasing the memory limit for the operator. Keep in mind that this value might be overwritten after a release update. + +We are working with the OpenShift team to resolve this issue. diff --git a/docs/content/installation/integrations/_index.md b/docs/content/installation/integrations/_index.md new file mode 100644 index 00000000000..e1c0b2f1033 --- /dev/null +++ b/docs/content/installation/integrations/_index.md @@ -0,0 +1,5 @@ +--- +title: Integrations +description: +weight: 600 +--- diff --git a/docs/content/app-protect-dos/_index.md b/docs/content/installation/integrations/app-protect-dos/_index.md similarity index 57% rename from docs/content/app-protect-dos/_index.md rename to docs/content/installation/integrations/app-protect-dos/_index.md index 9f9c0ba7eb7..2cd1a1fd305 100644 --- a/docs/content/app-protect-dos/_index.md +++ b/docs/content/installation/integrations/app-protect-dos/_index.md @@ -1,8 +1,8 @@ --- -title: Using with NGINX App Protect DoS +title: NGINX App Protect DoS description: Learn how to use NGINX Ingress Controller for Kubernetes with NGINX App Protect DoS. -weight: 1600 +weight: 200 menu: docs: - parent: NGINX Ingress Controller + parent: Integrations --- diff --git a/docs/content/app-protect-dos/configuration.md b/docs/content/installation/integrations/app-protect-dos/configuration.md similarity index 86% rename from docs/content/app-protect-dos/configuration.md rename to docs/content/installation/integrations/app-protect-dos/configuration.md index 3d7e2c59cf7..c2a9080bb8f 100644 --- a/docs/content/app-protect-dos/configuration.md +++ b/docs/content/installation/integrations/app-protect-dos/configuration.md @@ -2,19 +2,18 @@ title: Configuration description: "This document describes how to configure the NGINX App Protect Dos module." -weight: 1900 +weight: 200 doctypes: [""] toc: true docs: "DOCS-580" --- -This document describes how to configure the NGINX App Protect DoS module -> Check out the complete [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-dos). +> Check out the complete [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-dos). ## App Protect DoS Configuration A `DosProtectedResource` is a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) that holds the configuration of a collection of protected resources. -An [Ingress](/nginx-ingress-controller/configuration/ingress-resources/basic-configuration), [VirtualServer and VirtualServerRoute](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) can be protected by specifying a reference to the DosProtectedResource. +An [Ingress]({{< relref "configuration/ingress-resources/basic-configuration" >}}), [VirtualServer and VirtualServerRoute]({{< relref "configuration/virtualserver-and-virtualserverroute-resources.md" >}}) can be protected by specifying a reference to the DosProtectedResource. 1. Create an `DosProtectedResource` Custom resource manifest. As an example: @@ -109,7 +108,7 @@ Then add a reference in the `DosProtectedResource` to the `ApDosPolicy`: apDosPolicy: "default/dospolicy" ``` -## App Protect DoS Logs +## App Protect DoS Logs {#app-protect-dos-logs} You can set the [App Protect DoS Log configuration](/nginx-app-protect-dos/monitoring/types-of-logs/) by creating an `APDosLogConf` [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) and specifying the qualified identifier(`namespace/name`) of the `ApDosLogConf` in the `DosProtectedResource`. @@ -159,4 +158,4 @@ Then add a reference in the `DosProtectedResource` to the `APDosLogConf`: ## Global Configuration -The NGINX Ingress Controller has a set of global configuration parameters that align with those available in the NGINX App Protect DoS module. See [ConfigMap keys](/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#modules) for the complete list. The App Protect parameters use the `app-protect-dos*` prefix. +The NGINX Ingress Controller has a set of global configuration parameters that align with those available in the NGINX App Protect DoS module. See [ConfigMap keys]({{< relref "configuration/global-configuration/configmap-resource.md#modules" >}}) for the complete list. The App Protect parameters use the `app-protect-dos*` prefix. diff --git a/docs/content/app-protect-dos/dos-protected.md b/docs/content/installation/integrations/app-protect-dos/dos-protected.md similarity index 97% rename from docs/content/app-protect-dos/dos-protected.md rename to docs/content/installation/integrations/app-protect-dos/dos-protected.md index a9da72e34c1..d5cb4edcb81 100644 --- a/docs/content/app-protect-dos/dos-protected.md +++ b/docs/content/installation/integrations/app-protect-dos/dos-protected.md @@ -2,7 +2,7 @@ title: DoS Protected Resource description: "Dos Protected Resource Specification" -weight: 1800 +weight: 300 doctypes: [""] toc: true docs: "DOCS-581" @@ -40,7 +40,7 @@ spec: |``apDosMonitor.timeout`` | Determines how long (in seconds) should NGINX App Protect DoS wait for a response. [App Protect DoS monitor](#dosprotectedresourceapdosmonitor) Default value: 10 seconds for http1/http2 and 5 seconds for grpc. | ``int64`` | No | |``apDosPolicy`` | The [App Protect DoS policy](#dosprotectedresourceapdospolicy) of the dos. Accepts an optional namespace. | ``string`` | No | |``dosSecurityLog.enable`` | Enables security log. | ``bool`` | No | -|``dosSecurityLog.apDosLogConf`` | The [App Protect DoS log conf](/nginx-ingress-controller/app-protect-dos/configuration/#app-protect-dos-logs) resource. Accepts an optional namespace. | ``string`` | No | +|``dosSecurityLog.apDosLogConf`` | The [App Protect DoS log conf]({{< relref "installation/integrations/app-protect-dos/configuration.md#app-protect-dos-logs" >}}) resource. Accepts an optional namespace. | ``string`` | No | |``dosSecurityLog.dosLogDest`` | The log destination for the security log. Accepted variables are ``syslog:server=:``, ``stderr``, ````. Default is ``"syslog:server=127.0.0.1:514"``. | ``string`` | No | {{% /table %}} diff --git a/docs/content/installation/integrations/app-protect-dos/installation.md b/docs/content/installation/integrations/app-protect-dos/installation.md new file mode 100644 index 00000000000..700f330f900 --- /dev/null +++ b/docs/content/installation/integrations/app-protect-dos/installation.md @@ -0,0 +1,189 @@ +--- +title: Building NGINX Ingress Controller with NGINX App Protect DoS +description: "This document explains how to build an image for NGINX Ingress Controller with the NGINX App Protect DoS from source code." +weight: 100 +doctypes: [""] +toc: true +docs: "DOCS-583" +--- + +{{< custom-styles >}} + +{{}}If you'd rather not build your own NGINX Ingress Controller image, see the [pre-built image options](#pre-built-images) at the end of this guide.{{}} + +## Before you start + +- To use NGINX App Protect DoS with NGINX Ingress Controller, you must have NGINX Plus. + +--- + +## Prepare the environment {#prepare-environment} + +Get your system ready for building and pushing the NGINX Ingress Controller image with NGINX App Protect DoS. + +1. Sign in to your private registry. Replace `` with the path to your own private registry. + + ```shell + docker login + ``` + +2. Clone the NGINX Ingress Controller GitHub repository. Replace `` with the version of NGINX Ingress Controller you want. + + ```shell + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch + cd kubernetes-ingress + ``` + + For instance if you want to clone version v3.2.1, the commands to run would be: + + ```shell + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1 + cd kubernetes-ingress/deployments + ``` + +--- + +## Build the image {#build-docker-image} + +Follow these steps to build the NGINX Controller Image with NGINX App Protect DoS. + +1. Place your NGINX Plus license files (_nginx-repo.crt_ and _nginx-repo.key_) in the project's root folder. To verify they're in place, run: + + ```shell + ls nginx-repo.* + ``` + + You should see: + + ```shell + nginx-repo.crt nginx-repo.key + ``` + +2. Build the image. Replace `` with your chosen build option and `` with your private registry's path. Refer to the [Makefile targets](#makefile-targets) table below for the list of build options. + + ```shell + make PREFIX=/nginx-plus-ingress TARGET=download + ``` + + For example, to build a Debian-based image with NGINX Plus and NGINX App Protect DoS, run: + + ```shell + make debian-image-dos-plus PREFIX=/nginx-plus-ingress TARGET=download + ``` + + **What to expect**: The image is built and tagged with a version number, which is derived from the `VERSION` variable in the [_Makefile_]({{< relref "installation/building-nginx-ingress-controller.md#makefile-details" >}}). This version number is used for tracking and deployment purposes. + +{{}}In the event a patch version of NGINX Plus is released, make sure to rebuild your image to get the latest version. If your system is caching the Docker layers and not updating the packages, add `DOCKER_BUILD_OPTIONS="--pull --no-cache"` to the make command.{{}} + +### Makefile targets {#makefile-targets} + +{{}} +| Makefile Target | Description | Compatible Systems | +|---------------------------|-------------------------------------------------------------------|---------------------| +| **debian-image-dos-plus** | Builds a Debian-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module. | Debian | +| **debian-image-nap-dos-plus** | Builds a Debian-based image with NGINX Plus, [NGINX App Protect DoS](/nginx-app-protect-dos/), and [NGINX App Protect WAF](/nginx-app-protect/). | Debian | +| **ubi-image-dos-plus** | Builds a UBI-based image with NGINX Plus and the [NGINX App Protect DoS](/nginx-app-protect-dos/) module. | OpenShift | +| **ubi-image-nap-dos-plus** | Builds a UBI-based image with NGINX Plus, [NGINX App Protect DoS](/nginx-app-protect-dos/), and [NGINX App Protect WAF](/nginx-app-protect/). | OpenShift | +{{}} + +
+ +{{}}For the complete list of _Makefile_ targets and customizable variables, see the [Building NGINX Ingress Controller]({{< relref "installation/building-nginx-ingress-controller.md#makefile-details" >}}) guide{{}} + +--- + +## Push the image to your private registry + +Once you've successfully built the NGINX Ingress Controller image with NGINX App Protect DoS, the next step is to upload it to your private Docker registry. This makes the image available for deployment to your Kubernetes cluster. + +To upload the image, run the following command. If you're using a custom tag, add `TAG=your-tag` to the end of the command. Replace `` with your private registry's path. + +```shell +make push PREFIX=/nginx-plus-ingress +``` + +--- + +## Set up role-based access control (RBAC) {#set-up-rbac} + +{{< include "rbac/set-up-rbac.md" >}} + +--- + +## Create common resources {#create-common-resources} + +{{< include "installation/create-common-resources.md" >}} + +--- + +## Deploy NGINX Ingress Controller {#deploy-ingress-controller} + +You have two options for deploying NGINX Ingress Controller: + +- **Deployment**. Choose this method for the flexibility to dynamically change the number of NGINX Ingress Controller replicas. +- **DaemonSet**. Choose this method if you want NGINX Ingress Controller to run on all nodes or a subset of nodes. + +Before you start, update the [command-line arguments]({{< relref "configuration/global-configuration/command-line-arguments.md" >}}) for the NGINX Ingress Controller container in the relevant manifest file to meet your specific requirements. + +### Using a Deployment + +{{< include "installation/manifests/deployment.md" >}} + +### Using a DaemonSet + +{{< include "installation/manifests/daemonset.md" >}} + +--- + +## Install the App Protect DoS Arbitrator + +### Helm Chart + +The App Protect DoS Arbitrator can be installed using the [NGINX App Protect DoS Helm Chart](https://github.com/nginxinc/nap-dos-arbitrator-helm-chart). +If you have the NGINX Helm Repository already added, you can install the App Protect DoS Arbitrator by running the following command: + +```shell +helm install my-release-dos nginx-stable/nginx-appprotect-dos-arbitrator +``` + +### YAML Manifests + +Alternatively, you can install the App Protect DoS Arbitrator using the YAML manifests provided in the NGINX Ingress Controller repo. + +1. Create the namespace and service account: + + ```shell + kubectl apply -f common/ns-and-sa.yaml + ``` + +2. Deploy the NGINX App Protect Arbitrator as a Deployment and service: + + ```shell + kubectl apply -f deployment/appprotect-dos-arb.yaml + kubectl apply -f service/appprotect-dos-arb-svc.yaml + ``` + +--- + +## Enable NGINX App Protect DoS module + +To enable the NGINX App Protect DoS Module: + +- Add the `enable-app-protect-dos` [command-line argument]({{< relref "configuration/global-configuration/command-line-arguments.md#cmdoption-enable-app-protect-dos" >}}) to your Deployment or DaemonSet file. + +--- + +## Confirm NGINX Ingress Controller is running + +{{< include "installation/manifests/verify-pods-are-running.md" >}} + +For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-dos/configuration.md" >}}),the [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-dos). + +--- + +## Alternatives to building your own image {#pre-built-images} + +If you prefer not to build your own NGINX Ingress Controller image, you can use pre-built images. Here are your options: + +- Download the image using your NGINX Ingress Controller subscription certificate and key. See the [Getting the F5 Registry NGINX Ingress Controller Image]({{< relref "installation/nic-images/pulling-ingress-controller-image.md" >}}) guide. +- Use your NGINX Ingress Controller subscription JWT token to get the image: Instructions are in [Getting the NGINX Ingress Controller Image with JWT]({{< relref "installation/nic-images/using-the-jwt-token-docker-secret.md" >}}). diff --git a/docs/content/app-protect-waf/_index.md b/docs/content/installation/integrations/app-protect-waf/_index.md similarity index 80% rename from docs/content/app-protect-waf/_index.md rename to docs/content/installation/integrations/app-protect-waf/_index.md index 2f33c4e3e7a..1a1573fec02 100644 --- a/docs/content/app-protect-waf/_index.md +++ b/docs/content/installation/integrations/app-protect-waf/_index.md @@ -1,7 +1,7 @@ --- -title: Using with NGINX App Protect +title: NGINX App Protect WAF description: Learn how to use NGINX Ingress Controller for Kubernetes with NGINX App Protect. -weight: 1600 +weight: 100 aliases: ["/nginx-ingress-controller/app-protect/"] menu: docs: diff --git a/docs/content/app-protect-waf/configuration.md b/docs/content/installation/integrations/app-protect-waf/configuration.md similarity index 91% rename from docs/content/app-protect-waf/configuration.md rename to docs/content/installation/integrations/app-protect-waf/configuration.md index 7ac1d3d4078..35577695265 100644 --- a/docs/content/app-protect-waf/configuration.md +++ b/docs/content/installation/integrations/app-protect-waf/configuration.md @@ -8,19 +8,19 @@ docs: "DOCS-578" aliases: ["/app-protect/configuration/"] --- -> Check out the complete NGINX Ingress Controller with NGINX App Protect WAF example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). +> Check out the complete NGINX Ingress Controller with NGINX App Protect WAF example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). ## Global Configuration -NGINX Ingress Controller has a set of global configuration parameters that align with those available in NGINX App Protect WAF. See [ConfigMap keys](/nginx-ingress-controller/configuration/global-configuration/configmap-resource/#modules) for the complete list. The NGINX App Protect WAF parameters use the `app-protect*` prefix. +NGINX Ingress Controller has a set of global configuration parameters that align with those available in NGINX App Protect WAF. See [ConfigMap keys]({{< relref "configuration/global-configuration/configmap-resource.md#modules" >}}) for the complete list. The NGINX App Protect WAF parameters use the `app-protect*` prefix. ## Enable NGINX App Protect WAF You can enable and configure NGINX App Protect WAF on the Custom Resources (VirtualServer, VirtualServerRoute) or on the Ingress-resource basis. -To configure NGINX App Protect WAF on a VirtualServer resource, you would create a Policy Custom Resource referencing the APPolicy Custom Resource, and add this to the VirtualServer definition. See the documentation on the [NGINX App Protect WAF Policy](/nginx-ingress-controller/configuration/policy-resource/#waf). +To configure NGINX App Protect WAF on a VirtualServer resource, you would create a Policy Custom Resource referencing the APPolicy Custom Resource, and add this to the VirtualServer definition. See the documentation on the [NGINX App Protect WAF Policy]({{< relref "configuration/policy-resource.md#waf" >}}). -To configure NGINX App Protect WAF on an Ingress resource, you would apply the [`app-protect` annotations](/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#app-protect) to each desired resource. +To configure NGINX App Protect WAF on an Ingress resource, you would apply the [`app-protect` annotations]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations.md#app-protect" >}}) to each desired resource. ## NGINX App Protect WAF Policies @@ -108,7 +108,7 @@ To add the [log configurations](/nginx-app-protect-waf/logging-overview/security 1. Create an `APLogConf` Custom Resource manifest. 2. Add the desired log configuration to the `spec` field in the `APLogConf` resource. -3. Add the `APLogConf` reference to the [VirtualServer Policy resource](/nginx-ingress-controller/configuration/policy-resource/#waf) or the [Ingress resource](/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#app-protect) as per the documentation. +3. Add the `APLogConf` reference to the [VirtualServer Policy resource]({{< relref "configuration/policy-resource.md#waf" >}}) or the [Ingress resource]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations.md#app-protect" >}}) as per the documentation. > **Note**: The fields from the JSON must be presented in the YAML *exactly* the same, in name and level. NGINX Ingress Controller will transform the YAML into a valid JSON WAF log config. @@ -149,7 +149,7 @@ You can define NGINX App Protect WAF [User-Defined Signatures](/nginx-app-protec > **Note**: The field `revisionDatetime` is not currently supported. -> **Note**: `APUserSig` resources increase the reload time of NGINX Plus compared with `APPolicy` and `APLogConf` resources. Refer to [NGINX Fails to Start or Reload](/nginx-ingress-controller/app-protect/troubleshooting/#nginx-fails-to-start-or-reload) for more information. +> **Note**: `APUserSig` resources increase the reload time of NGINX Plus compared with `APPolicy` and `APLogConf` resources. Refer to [NGINX Fails to Start or Reload]({{< relref "troubleshooting/troubleshooting-app-protect-waf.md#nginx-fails-to-start-or-reload" >}}) for more information. To add the [User Defined Signatures](https://docs.nginx.com/nginx-app-protect-waf/configuration-guide/configuration/#user-defined-signatures) to a VirtualServer or Ingress resource: @@ -245,8 +245,8 @@ These are the typical steps to deploy an OpenAPI protection Policy in NGINX Ingr 3. Make other custom changes if needed (e.g. enable Data Guard protection). 4. Use a tool to convert the result to YAML. There are many, for example: [`yq` utility](https://github.com/mikefarah/yq). 5. Add the YAML properties to create an `APPolicy` Custom Resource putting the policy itself (as in step 4) within the `spec` property of the Custom Resource. Refer to the [NGINX App Protect Policies](#nginx-app-protect-waf-policies) section above. -6. Create a `Policy` object which references the `APPolicy` Custom Resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/app-protect-waf/waf.yaml). -7. Finally, attach the `Policy` object to a `VirtualServer` resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/app-protect-waf/virtual-server.yaml). +6. Create a `Policy` object which references the `APPolicy` Custom Resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/app-protect-waf/waf.yaml). +7. Finally, attach the `Policy` object to a `VirtualServer` resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/app-protect-waf/virtual-server.yaml). **Note**: You need to make sure that the server where the resource files are located is always available when you are compiling your policy. @@ -381,7 +381,7 @@ The `link` option is also available in the `openApiFileReference` property and i In this example we deploy NGINX Ingress Controller with NGINX Plus and NGINX App Protect WAF, deploy a simple web application, and then configure load balancing and WAF protection for that application using the VirtualServer resource. -**Note:** You can find the example, and the files referenced, on [GitHub](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-waf). +**Note:** You can find the example, and the files referenced, on [GitHub](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-waf). ## Prerequisites @@ -403,7 +403,7 @@ In this example we deploy NGINX Ingress Controller with NGINX Plus and NGINX App Create the application deployment and service: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/webapp.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/webapp.yaml ``` ### Step 2. Deploy the AP Policy @@ -411,15 +411,15 @@ Create the application deployment and service: 1. Create the syslog service and pod for the NGINX App Protect WAF security logs: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/syslog.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/syslog.yaml ``` 2. Create the User-Defined Signature, WAF policy, and log configuration: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/ap-apple-uds.yaml - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/ap-dataguard-alarm-policy.yaml - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/ap-logconf.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/ap-apple-uds.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/ap-dataguard-alarm-policy.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/ap-logconf.yaml ``` ### Step 3 - Deploy the WAF Policy @@ -427,7 +427,7 @@ Create the application deployment and service: Create the WAF policy ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/waf.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/waf.yaml ``` Note the NGINX App Protect WAF configuration settings in the Policy resource. They enable WAF protection by configuring NGINX App Protect WAF with the policy and log configuration created in the previous step. @@ -437,7 +437,7 @@ Create the WAF policy 1. Create the VirtualServer Resource: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/virtual-server.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/virtual-server.yaml ``` Note that the VirtualServer references the policy waf-policy created in Step 3. @@ -478,7 +478,7 @@ To access the application, curl the coffee and the tea services. We'll use the - ### Configuration Example of Virtual Server -Refer to GitHub repo for [Virtual Server example](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/webapp.yaml). +Refer to GitHub repo for [Virtual Server example](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/webapp.yaml). ```yaml apiVersion: k8s.nginx.org/v1 diff --git a/docs/content/app-protect-waf/installation.md b/docs/content/installation/integrations/app-protect-waf/installation.md similarity index 65% rename from docs/content/app-protect-waf/installation.md rename to docs/content/installation/integrations/app-protect-waf/installation.md index 664353ac6b0..a29038920de 100644 --- a/docs/content/app-protect-waf/installation.md +++ b/docs/content/installation/integrations/app-protect-waf/installation.md @@ -12,18 +12,18 @@ aliases: ["/app-protect/installation/"] This document provides an overview of the steps required to use NGINX App Protect WAF with your NGINX Ingress Controller deployment. You can visit the linked documents to find additional information and instructions. -You can also [install the Ingress Controller with App Protect WAF by using Helm](/nginx-ingress-controller/installation/installation-with-helm/). Use the `controller.appprotect.*` parameters of the chart. +You can also [install the Ingress Controller with App Protect WAF by using Helm]({{< relref "installation/installing-nic/installation-with-helm.md" >}}). Use the `controller.appprotect.*` parameters of the chart. ## Prerequisites 1. Make sure you have access to the Ingress Controller image: - - For NGINX Plus Ingress Controller, see [here](/nginx-ingress-controller/installation/pulling-ingress-controller-image) for details on how to pull the image from the F5 Docker registry. - - To pull from the F5 Container registry in your Kubernetes cluster, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here](/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret). - - It is also possible to build your own image and push it to your private Docker registry by following the instructions from [here](/nginx-ingress-controller/installation/building-ingress-controller-image). + - For NGINX Plus Ingress Controller, see [here]({{< relref "installation/nic-images/pulling-ingress-controller-image" >}}) for details on how to pull the image from the F5 Docker registry. + - To pull from the F5 Container registry in your Kubernetes cluster, configure a docker registry secret using your JWT token from the MyF5 portal by following the instructions from [here]({{< relref "installation/nic-images/using-the-jwt-token-docker-secret" >}}). + - It is also possible to build your own image and push it to your private Docker registry by following the instructions from [here]({{< relref "installation/building-nginx-ingress-controller.md" >}})). 2. Clone the Ingress Controller repo: ```console - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1 cd kubernetes-ingress ``` @@ -31,7 +31,7 @@ You can also [install the Ingress Controller with App Protect WAF by using Helm] Take the steps below to create the Docker image that you'll use to deploy NGINX Ingress Controller with App Protect in Kubernetes. -- [Build the NGINX Ingress Controller image](/nginx-ingress-controller/installation/building-ingress-controller-image). +- [Build the NGINX Ingress Controller image]({{< relref "installation/building-nginx-ingress-controller.md" >}})). When running the `make` command to build the image, be sure to use the `debian-image-nap-plus` target. For example: @@ -51,18 +51,18 @@ Take the steps below to create the Docker image that you'll use to deploy NGINX **Note**: In the event of a patch version of NGINX Plus being [released](/nginx/releases/), make sure to rebuild your image to get the latest version. The Dockerfile will use the latest available version of the [Attack Signatures](/nginx-app-protect/configuration/#attack-signatures) and [Threat Campaigns](/nginx-app-protect/configuration/#threat-campaigns) packages at the time of build. If your system is caching the Docker layers and not updating the packages, add `DOCKER_BUILD_OPTIONS="--no-cache"` to the `make` command. -- [Push the image to your local Docker registry](/nginx-ingress-controller/installation/building-ingress-controller-image/#building-the-image-and-pushing-it-to-the-private-registry). +- [Push the image to your local Docker registry]({{< relref "installation/building-nginx-ingress-controller.md#build-image-push-to-private-repo" >}}. ## Install the Ingress Controller Take the steps below to set up and deploy the NGINX Ingress Controller and App Protect WAF module in your Kubernetes cluster. -1. [Configure role-based access control (RBAC)](/nginx-ingress-controller/installation/installation-with-manifests/#1-configure-rbac). +1. [Configure role-based access control (RBAC)]({{< relref "installation/installing-nic/installation-with-manifests.md#configure-rbac" >}}). > **Important**: You must have an admin role to configure RBAC in your Kubernetes cluster. -2. [Create the common Kubernetes resources](/nginx-ingress-controller/installation/installation-with-manifests/#2-create-common-resources). -3. Enable the App Protect WAF module by adding the `enable-app-protect` [cli argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#cmdoption-enable-app-protect) to your Deployment or DaemonSet file. -4. [Deploy the Ingress Controller](/nginx-ingress-controller/installation/installation-with-manifests/#3-deploy-the-ingress-controller). +2. [Create the common Kubernetes resources]({{< relref "installation/installing-nic/installation-with-manifests.md#create-common-resources" >}}). +3. Enable the App Protect WAF module by adding the `enable-app-protect` [cli argument]({{< relref "configuration/global-configuration/command-line-arguments.md#cmdoption-enable-app-protect" >}}) to your Deployment or DaemonSet file. +4. [Deploy the Ingress Controller]({{< relref "installation/installing-nic/installation-with-manifests.md#deploy-ingress-controller" >}}). -For more information, see the [Configuration guide](/nginx-ingress-controller/app-protect/configuration) and the NGINX Ingress Controller with App Protect example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). +For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-waf/configuration.md" >}}) and the NGINX Ingress Controller with App Protect example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf" >}}). diff --git a/docs/content/installation/integrations/f5-ingresslink.md b/docs/content/installation/integrations/f5-ingresslink.md new file mode 100644 index 00000000000..e1735fcdfbe --- /dev/null +++ b/docs/content/installation/integrations/f5-ingresslink.md @@ -0,0 +1,94 @@ +--- +title: F5 BIG-IP +description: | + Learn how to use NGINX Ingress Controller with F5 IngressLink to configure your F5 BIG-IP device. +weight: 300 +doctypes: ["concept"] +toc: true +docs: "DOCS-600" +--- + +F5 IngressLink is the integration between NGINX Ingress Controller and [F5 BIG-IP Container Ingress Services](https://clouddocs.f5.com/containers/latest/) (CIS) that configures an F5 BIG-IP device as a load balancer for NGINX Ingress Controller pods. + +## Install NGINX Ingress Controller with the integration enabled + +The steps to enable the integration depend on the option chosen to install NGINX Ingress Controller: Using [Manifests]({{< relref "installation/installing-nic/installation-with-manifests" >}}) or using the [Helm chart]({{< relref "installation/installing-nic/installation-with-helm" >}}). + +### Installation using manifests + +1. Create a service for the Ingress Controller pods for ports 80 and 443. For example: + + ```yaml + apiVersion: v1 + kind: Service + metadata: + name: nginx-ingress-ingresslink + namespace: nginx-ingress + labels: + app: ingresslink + spec: + ports: + - port: 80 + targetPort: 80 + protocol: TCP + name: http + - port: 443 + targetPort: 443 + protocol: TCP + name: https + selector: + app: nginx-ingress + ``` + + Note the label `app: ingresslink`. We will use it in the [Configure CIS](#configure-cis) step. + +1. In the [ConfigMap resource]({{< relref "configuration/global-configuration/configmap-resource" >}}) enable the proxy protocol, which the BIG-IP system will use to pass the client IP and port information to NGINX. For the `set-real-ip-from` key, use the subnet of the IP which the BIG-IP system uses to send traffic to NGINX: + + ```yaml + proxy-protocol: "True" + real-ip-header: "proxy_protocol" + set-real-ip-from: "0.0.0.0/0" + ``` + +1. Deploy NGINX Ingress Controller with additional [command-line arguments]({{< relref "configuration/global-configuration/command-line-arguments" >}}): + + ```yaml + args: + - -ingresslink=nginx-ingress + - -report-ingress-status + . . . + ``` + + where `ingresslink` references the name of the IngressLink resource from step 1, and `report-ingress-status` enables [reporting ingress statuses]({{< relref "configuration/global-configuration/reporting-resources-status#ingress-resources" >}}). + +### Installation using Helm + +Install a Helm release with the following values: + +```yaml +controller: + config: + entries: + proxy-protocol: "True" + real-ip-header: "proxy_protocol" + set-real-ip-from: "0.0.0.0/0" + reportIngressStatus: + ingressLink: nginx-ingress + service: + type: ClusterIP + externalTrafficPolicy: Cluster + extraLabels: + app: ingresslink +``` + +We will use the `ingressLink` and `extraLabels` parameter values to configure CIS in the next section. For the `set-real-ip-from` key, use the subnet of the IP which the BIG-IP system uses to send traffic to NGINX. + +## Configure CIS + +To enable the integration, F5 BIG-IP Container Ingress Services must be deployed in the cluster and configured to support the integration. Follow the instructions on the [CIS documentation portal](https://clouddocs.f5.com/containers/latest/userguide/ingresslink/#configuring-ingresslink). + +Make sure that: + +- The name of the IngressLink resource is the same as the one used during the installation of NGINX Ingress Controller (`nginx-ingress` in the previous example). +- The selector in the IngressLink resource is the same as the Service labels configured during Ingress Controller installation (`app: ingresslink` in the previous example). +- The IngressLink must belong to the same namespace as the Ingress Controller pod (`nginx-ingress` or the namespace used for installing the Helm chart). diff --git a/docs/content/installation/integrations/opentracing.md b/docs/content/installation/integrations/opentracing.md new file mode 100644 index 00000000000..6572fe1f704 --- /dev/null +++ b/docs/content/installation/integrations/opentracing.md @@ -0,0 +1,113 @@ +--- +title: OpenTracing +description: "Learn how to use OpenTracing with NGINX Ingress Controller." +weight: 400 +doctypes: [""] +aliases: + - /opentracing/ +toc: true +docs: "DOCS-618" +--- + + +NGINX Ingress Controller supports [OpenTracing](https://opentracing.io/) with the third-party module [opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing). + +## Prerequisites + +1. Use an Ingress Controller image that contains OpenTracing. + + - You can find the images that include OpenTracing listed [in the technical specs doc]({{< relref "technical-specifications.md#supported-docker-images" >}}). + - Alternatively, you can [build your own image]({{< relref "installation/building-nginx-ingress-controller.md" >}}) using `debian-image` (or `alpine-image`) for NGINX or `debian-image-plus` (or `alpine-image-plus`) for NGINX Plus. + - [Jaeger](https://github.com/jaegertracing/jaeger-client-cpp), [Zipkin](https://github.com/rnburn/zipkin-cpp-opentracing) and [Datadog](https://github.com/DataDog/dd-opentracing-cpp/) tracers are installed by default. + +1. Enable snippets annotations by setting the [`enable-snippets`]({{< relref "configuration/global-configuration/command-line-arguments#cmdoption-enable-snippets" >}}) command-line argument to true. + +1. Load the OpenTracing module. + + You need to load the module with the configuration for the chosen tracer using the following ConfigMap keys: + + - `opentracing-tracer`: sets the path to the vendor tracer binary plugin. This is the path you used in the COPY line of step *ii* above. + - `opentracing-tracer-config`: sets the tracer configuration in JSON format. + + The following example shows how to use these two keys to load the module with Jaeger tracer: + + ```yaml + opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so" + opentracing-tracer-config: | + { + "service_name": "nginx-ingress", + "propagation_format": "w3c", + "sampler": { + "type": "const", + "param": 1 + }, + "reporter": { + "localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831" + } + } + ``` + +## Enable OpenTracing globally + +To enable OpenTracing globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `opentracing` ConfigMap key to `True`: + +```yaml +opentracing: True +``` + +## Enable or disable OpenTracing per Ingress resource + +You can use annotations to enable or disable OpenTracing for a specific Ingress resource. As mentioned in the prerequisites section, both `opentracing-tracer` and `opentracing-tracer-config` must be configured. + +Consider the following two cases: + +### OpenTracing is globally disabled + +1. To enable OpenTracing for a specific Ingress resource, use the server snippet annotation: + + ```yaml + nginx.org/server-snippets: | + opentracing on; + ``` + +1. To enable OpenTracing for specific paths: + + - You need to use [Mergeable Ingress resources]({{< relref "configuration/ingress-resources/cross-namespace-configuration" >}}) + - You need to use the location snippets annotation to enable OpenTracing for the paths of a specific Minion Ingress resource: + + ```yaml + nginx.org/location-snippets: | + opentracing on; + ``` + +### OpenTracing is globally enabled + +1. To disable OpenTracing for a specific Ingress resource, use the server snippet annotation: + + ```yaml + nginx.org/server-snippets: | + opentracing off; + ``` + +1. To disable OpenTracing for specific paths: + + - You need to use [Mergeable Ingress resources]({{< relref "configuration/ingress-resources/cross-namespace-configuration" >}}) + - You need to use the location snippets annotation to disable OpenTracing for the paths of a specific Minion Ingress resource: + + ```yaml + nginx.org/location-snippets: | + opentracing off; + ``` + +## Customize OpenTracing + +You can customize OpenTracing through the supported [OpenTracing module directives](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md). Use the location-snippets ConfigMap keys or annotations to insert those directives into the generated NGINX configuration. + +For example, to propagate the active span context for upstream requests, you need to set the `opentracing_propagate_context` directive, which you can add to an Ingress resource using the location snippets annotation: + +```yaml +nginx.org/location-snippets: | + opentracing_propagate_context; +``` + +{{< note >}}The `opentracing_propagate_context` and `opentracing_grpc_propagate_context` directives can be used in `http`, `server` or `location` contexts according to the [module documentation](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md#opentracing_propagate_context). However, because of the way the module works and how NGINX Ingress Controller generates the NGINX configuration, it is only possible to use the directive in the `location` context.{{< /note >}} diff --git a/docs/content/installation/nic-images/_index.md b/docs/content/installation/nic-images/_index.md new file mode 100644 index 00000000000..1d740ffaaa4 --- /dev/null +++ b/docs/content/installation/nic-images/_index.md @@ -0,0 +1,5 @@ +--- +title: NGINX Ingress Controller Images +description: +weight: 300 +--- diff --git a/docs/content/installation/nic-images/pulling-ingress-controller-image.md b/docs/content/installation/nic-images/pulling-ingress-controller-image.md new file mode 100644 index 00000000000..5abdf42f929 --- /dev/null +++ b/docs/content/installation/nic-images/pulling-ingress-controller-image.md @@ -0,0 +1,158 @@ +--- +title: Getting the F5 Registry NGINX Ingress Controller Image +description: "Learn how to pull an F5 NGINX Plus Ingress Controller image—including those with NGINX App Protect WAF and NGINX App Protect DoS—from the official F5 Docker registry and upload it to your private registry. This guide covers the prerequisites, image tagging, and troubleshooting steps." +weight: 100 +doctypes: ["install"] +toc: true +docs: "DOCS-605" +--- + +{{}} + +--- + +## Before you begin + +Before you start, you'll need these installed on your machine: + +- [Docker v18.09 or higher](https://docs.docker.com/engine/release-notes/18.09/). +- An NGINX Ingress Controller subscription. Download both the certificate (*nginx-repo.crt*) and key (*nginx-repo.key*) from the [MyF5 Customer Portal](https://my.f5.com). Keep in mind that an NGINX Plus certificate and key won't work for for the steps in this guide. + +--- + +## Set up Docker for F5 Container Registry + +Start by setting up Docker to communicate with the F5 Container Registry located at `private-registry.nginx.com`. If you're using Linux, follow these steps to create a directory and add your certificate and key: + +```shell +mkdir -p /etc/docker/certs.d/private-registry.nginx.com +cp /etc/docker/certs.d/private-registry.nginx.com/client.cert +cp /etc/docker/certs.d/private-registry.nginx.com/client.key +``` + +The steps provided are for Linux. For Mac or Windows, consult the [Docker for Mac](https://docs.docker.com/docker-for-mac/#add-client-certificates) or [Docker for Windows](https://docs.docker.com/docker-for-windows/#how-do-i-add-client-certificates) documentation. For more details on Docker Engine security, you can refer to the [Docker Engine Security documentation](https://docs.docker.com/engine/security/). + +--- + +## Pull the image + +Next, pull the image you need from `private-registry.nginx.com`. To find the correct image, consult the [Tech Specs guide]({{< relref "technical-specifications#images-with-nginx-plus" >}}). + +To pull an image, follow these steps. Replace `` with the specific version you need, for example, `3.2.1`. + +- For NGINX Plus Ingress Controller, run: + + ```shell + docker pull private-registry.nginx.com/nginx-ic/nginx-plus-ingress: + ``` + +- For NGINX Plus Ingress Controller with NGINX App Protect WAF, run: + + ```shell + docker pull private-registry.nginx.com/nginx-ic-nap/nginx-plus-ingress: + ``` + +- For NGINX Plus Ingress Controller with NGINX App Protect DoS, run: + + ```shell + docker pull private-registry.nginx.com/nginx-ic-dos/nginx-plus-ingress: + ``` + +You can use the Docker registry API to list the available image tags by running the following commands. Replace `` with the location of your client key and `` with the location of your client certificate. The `jq` command is used to format the JSON output for easier reading. + +```json +$ curl https://private-registry.nginx.com/v2/nginx-ic/nginx-plus-ingress/tags/list --key --cert | jq +{ + "name": "nginx-ic/nginx-plus-ingress", + "tags": [ + "3.2.1-alpine", + "3.2.1-ubi", + "3.2.1" + ] +} + +$ curl https://private-registry.nginx.com/v2/nginx-ic-nap/nginx-plus-ingress/tags/list --key --cert | jq +{ + "name": "nginx-ic-nap/nginx-plus-ingress", + "tags": [ + "3.2.1-ubi", + "3.2.1" + ] +} + +$ curl https://private-registry.nginx.com/v2/nginx-ic-dos/nginx-plus-ingress/tags/list --key --cert | jq +{ + "name": "nginx-ic-dos/nginx-plus-ingress", + "tags": [ + "3.2.1-ubi", + "3.2.1" + ] +} +``` + +--- + +## Push to your private registry + +After pulling the image, tag it and upload it to your private registry. + +1. Log in to your private registry: + + ```shell + docker login + ``` + +1. Tag and push the image. Replace `` with your registry's path and `` with the version you're using, for example `3.2.1`: + + - For NGINX Plus Ingress Controller, run: + + ```shell + docker tag private-registry.nginx.com/nginx-ic/nginx-plus-ingress: /nginx-ic/nginx-plus-ingress: + docker push /nginx-ic/nginx-plus-ingress: + ``` + + - For NGINX Controller with NGINX App Protect WAF, run: + + ```shell + docker tag private-registry.nginx.com/nginx-ic-nap/nginx-plus-ingress: /nginx-ic-nap/nginx-plus-ingress: + docker push /nginx-ic-nap/nginx-plus-ingress: + ``` + + - For NGINX Controller with NGINX App Protect DoS, run: + + ```shell + docker tag private-registry.nginx.com/nginx-ic-dos/nginx-plus-ingress: /nginx-ic-dos/nginx-plus-ingress: + docker push /nginx-ic-dos/nginx-plus-ingress: + ``` + +--- + +## Troubleshooting + +If you encounter issues while following this guide, here are solutions to common problems: + +- **Certificate errors** + - **Likely Cause**: Incorrect certificate or key location, or using an NGINX Plus certificate. + - **Solution**: Verify you have the correct NGINX Ingress Controller certificate and key. Place them in the correct directory and ensure the certificate has a *.cert* extension. + +- **Docker version compatibility** + - **Likely Cause**: Outdated Docker version. + - **Solution**: Make sure you're running [Docker v18.09 or higher](https://docs.docker.com/engine/release-notes/18.09/). Upgrade if necessary. + +- **Can't pull the image** + - **Likely Cause**: Mismatched image name or tag. + - **Solution**: Double-check the image name and tag against the [Tech Specs guide]({{< relref "technical-specifications.md#images-with-nginx-plus" >}}). + +- **Failed to push to private registry** + - **Likely Cause**: Not logged into your private registry or incorrect image tagging. + - **Solution**: Verify login status and correct image tagging before pushing. Consult the [Docker documentation](https://docs.docker.com/docker-hub/repos/) for more details. + +--- + +## Alternative installation options + +You can also get the NGINX Ingress Controller image using the following alternate methods: + +- [Install using a JWT token in a Docker Config Secret]({{< relref "using-the-jwt-token-docker-secret" >}}). +- [Build the Ingress Controller image]({{< relref "building-nginx-ingress-controller" >}}) using the source code from the GitHub repository and your NGINX Plus subscription certificate and key. +- For NGINX Ingress Controller based on NGINX OSS, you can pull the [nginx/nginx-ingress image](https://hub.docker.com/r/nginx/nginx-ingress/) from DockerHub. diff --git a/docs/content/installation/nic-images/using-aws-marketplace-image.md b/docs/content/installation/nic-images/using-aws-marketplace-image.md new file mode 100644 index 00000000000..461ecc8f2c2 --- /dev/null +++ b/docs/content/installation/nic-images/using-aws-marketplace-image.md @@ -0,0 +1,89 @@ +--- +title: Using the AWS Marketplace NGINX Ingress Controller Image +description: "This guide walks you through the steps to set up NGINX Ingress Controller using the AWS Marketplace." +weight: 200 +doctypes: [""] +toc: true +docs: "DOCS-607" +--- + +## Before you being + +Follow this guide to set up NGINX Ingress Controller using AWS Marketplace. This involves some extra steps to make sure everything works as it should. + +{{< important >}}This guide focuses on EKS version 1.19. For EKS versions below 1.19, you'll need to adjust security settings in the NGINX Pod to ensure compatibility with marketplace images. Make sure you're using updated versions of `eksctl` and the AWS CLI.{{< /important >}} + +{{< note >}}AWS Region US-West-1 doesn't support NGINX Ingress Controller.{{}} + +## Instructions + +1. First, make sure your AWS EKS cluster is operational. If not, set one up using the AWS console or the `eksctl` tool. For step-by-step instructions, follow [this guide](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html). + +2. Create a new IAM role that will link to the service zccount for NGINX Ingress Controller. This role should have a policy that lets you monitor AWS NGINX Ingress Controller usage. Skipping this step will cause AWS NGINX Ingress Controller not to work. For more information, consult [AWS EKS IAM documentation](https://docs.aws.amazon.com/eks/latest/userguide/create-service-account-iam-policy-and-role.html) and [AWS Marketplace policy details](https://docs.aws.amazon.com/marketplace/latest/userguide/iam-user-policy-for-aws-marketplace-actions.html). + +3. Link this IAM role to your EKS cluster service account. Doing this will annotate your service account Kubernetes object with the IAM role link. + +{{< important >}}Associating your AWS EKS cluster with an OIDC provider is a prerequisite for creating your IAM service account.{{< /important >}} + +## Step-by-step instructions using eksctl + +Make sure you have an operational EKS cluster and that the namespace for your NGINX Ingress Controller is set up. If you don't have an EKS cluster yet, you'll need to create one. + +1. Associate your EKS cluster with an OIDC IAM provider. Use your specific `--cluster and `--region ` values. + + ``` shell + eksctl utils associate-iam-oidc-provider --region=eu-west-1 --cluster=json-eu-east1 --approve + ``` + +2. Create an IAM role and a service account for your cluster. Replace `--name `, `--namespace `, and `--region ` with your values. + + ``` shell + eksctl create iamserviceaccount --name nginx-ingress --namespace nginx-ingress --cluster json-test01 --region us-east-2 --attach-policy-arn arn:aws:iam::aws:policy/AWSMarketplaceMeteringRegisterUsage --approve + ``` + + This step creates the IAM role with the required policy, creates the service account if it doesn't exist, and adds the annotations needed for your AWS cluster. For additional details, consult the [AWS documentation](https://docs.aws.amazon.com/eks/latest/userguide/create-service-account-iam-policy-and-role.html). You don't need to apply any service account YAML files because `eksctl` handles that for you. + + ``` yaml + apiVersion: v1 + kind: ServiceAccount + metadata: + annotations: + EKS.amazonaws.com/role-arn: arn:aws:iam::001234567890:role/eksctl-json-us-west2-addon-iamserviceaccount-Role1-IJJ6CF9Y8IPY + labels: + app.kubernetes.io/managed-by: eksctl + name: nginx-ingress + namespace: nginx-ingress + secrets: + - name: nginx-ingress-token-zm728 + ``` + +
+ + Ensure the service account name matches the one in your _rbac.yaml_ file for manifest deployment. + + Here's what a sample _rbac.yaml_ file might look like: + + ``` yaml + kind: ClusterRoleBinding + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + name: nginx-ingress + subjects: + - kind: ServiceAccount + name: nginx-ingress + namespace: nginx-ingress + roleRef: + kind: ClusterRole + name: nginx-ingress + apiGroup: rbac.authorization.k8s.io + ``` + +3. Sign in to the AWS ECR registry that specified in the instructions on the [AWS Marketplace portal](https://aws.amazon.com/marketplace/pp/prodview-fx3faxl7zqeau?sr=0-1&ref_=beagle&applicationId=AWSMPContessa). + + {{< img title="ECR pull instructions for NGINX Ingress Controller" src="./img/ecr-pull-instructions.png" >}} + +
+ + {{< tip >}}For help with credentials, AWS Labs offers a credential helper. Check out [their GitHub repository](https://github.com/awslabs/amazon-ecr-credential-helper) for setup instructions.{{< /tip >}} + +4. Update the image in the _nginx-plus-ingress.yaml_ manifest. diff --git a/docs/content/installation/nic-images/using-gcp-marketplace-package.md b/docs/content/installation/nic-images/using-gcp-marketplace-package.md new file mode 100644 index 00000000000..3f61dd40b26 --- /dev/null +++ b/docs/content/installation/nic-images/using-gcp-marketplace-package.md @@ -0,0 +1,115 @@ +--- +title: Using the GCP Marketplace NGINX Ingress Controller Image +description: "Follow these steps to deploy F5 NGINX Ingress Controller through the GCP Marketplace." +weight: 300 +doctypes: [""] +toc: true +docs: "DOCS-000" +--- + +## Overview + +NGINX Ingress Controller acts as a Kubernetes Ingress Controller for both NGINX and NGINX Plus. It offers: + +- Routing based on host headers: For example, _foo.example.com_ routes to one set of services, while _bar.example.com_ routes to another. +- Path-based routing: Requests beginning with `/serviceA` go to service A; those starting with `/serviceB` go to service B. +- SSL/TLS termination for individual hostnames, like _foo.example.com_. + +## Before you begin + +Before installing NGINX Ingress Controller, review our [Installation with Manifests]({{< relref "installation/installing-nic/installation-with-manifests.md" >}}) guide. This guide shows you how to build a local NGINX Ingress Controller image and set up the required CustomResourceDefinitions (CRDs). + +## Installation + +Choose one of the following methods to install NGINX Ingress Controller. + +### Install NGINX Ingress Controller to an existing GKE cluster + +1. Open [Google Cloud Console](https://console.cloud.google.com/) and go to **Kubernetes Engine > Applications**. + +2. Select **DEPLOY FROM MARKETPLACE** and search for *NGINX Ingress Controller*. + + {{}}Make sure to choose a _Premium Edition_ image from _NGINX, Inc._, not a third-party one.{{}} + +3. Choose the appropriate *NGINX Ingress Controller* image, then select **CONFIGURE**. + +4. Choose your cluster: + + Select an existing Kubernetes Cluster from the list. The _default_ namespace is automatically chosen, but you can create a new one if you prefer. The **App instance name** will be a prefix for all resources created by the deployment and needs to be unique within the selected namespace. + + Recommended settings are pre-selected but feel free to adjust them. + + {{< note >}}If you see the **CREATE NEW CLUSTER** button, select **OR SELECT AN EXISTING CLUSTER** . {{}} + +5. Select **DEPLOY** to start NGINX Ingress Controller installation process. + + You can find NGINX Ingress Controller application by going back to **Kubernetes Engine > Applications**. + +### Install to a new GKE cluster + +If you prefer to use a new GKE cluster, follow these steps. Ensure you have enough vCPU for both NGINX Ingress Controller and any other applications you'll deploy. + +1. Open [Google Cloud Console](https://console.cloud.google.com/) and go to **Marketplace*. +2. Search for *NGINX Ingress Controller*. + + {{}}Make sure to choose a _Premium Edition_ image from _NGINX, Inc._, not a third-party one.{{}} + +3. Choose the appropriate *NGINX Ingress Controller* image, then select **CONFIGURE**. + +4. Configure the new GKE cluster: + + Choose the appropriate zone, network, and subnetwork. + +5. Select **CREATE NEW CLUSTER**. + + After a short delay, the cluster will be ready. + +6. Finish the installation: + + The _default_ namespace is automatically selected, but you can create a new one if you wish. The **App instance name** will be a prefix for all resources created by the deployment and needs to be unique within the selected namespace. Confirm or adjust the settings and then select **DEPLOY**. + + You can find your NGINX Ingress Controller application by going back to **Kubernetes Engine > Applications**. + +## Configuration + +When you install NGINX Ingress Controller from the GCP Marketplace, it comes with default settings and an empty *ConfigMap*. The resources have names ending in a suffix that reflects the app instance name you chose during installation. This suffix has the format -nginx-ingress. + +For example, if you've installed NGINX Ingress Controller in the `nginx-ingress` namespace and used the app instance name `nginx-ingress-plus`, you can check its _ConfigMap_ by running this `kubectl` command: + +```bash +kubectl get configmap -n nginx-ingress nginx-ingress-plus-nginx-ingress -o yaml +``` + +``` yaml +$ kubectl get configmap -n nginx-ingress nginx-ingress-plus-nginx-ingress -o yaml +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"v1","data":null,"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"nginx-ingress-plus","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/name":"nginx-ingress-plus-nginx-ingress","helm.sh/chart":"nginx-ingress-0.16.2"},"name":"nginx-ingress-plus-nginx-ingress","namespace":"nginx-ingress","ownerReferences":[{"apiVersion":"app.k8s.io/v1beta1","blockOwnerDeletion":true,"kind":"Application","name":"nginx-ingress-plus","uid":"5cbbebd8-df13-4001-bd65-9467405d9a9d"}]}} + creationTimestamp: "2022-08-25T01:03:10Z" + labels: + app.kubernetes.io/instance: nginx-ingress-plus + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: nginx-ingress-plus-nginx-ingress + helm.sh/chart: nginx-ingress-0.16.2 + name: nginx-ingress-plus-nginx-ingress + namespace: nginx-ingress + ownerReferences: + - apiVersion: app.k8s.io/v1beta1 + blockOwnerDeletion: true + kind: Application + name: nginx-ingress-plus + uid: 5cbbebd8-df13-4001-bd65-9467405d9a9d + resourceVersion: "147519" + uid: 3fa33891-7a30-4004-91bd-bd5d652e34a9 +``` + +
+ +For options to customize your resources, see our [Configuration documentation]({{< relref "configuration/" >}}). + +## Basic Usage + +To learn how to set up a basic application with NGINX Ingress Controller, refer to our [Basic Configuration Example](https://github.com/nginxinc/kubernetes-ingress/tree/main/examples/custom-resources/basic-configuration). diff --git a/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md b/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md new file mode 100644 index 00000000000..0efd40249c1 --- /dev/null +++ b/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md @@ -0,0 +1,198 @@ +--- +title: Getting the NGINX Ingress Controller Image with JWT +description: "Use your NGINX Ingress Controller subscription JWT token to get the NGINX Plus Ingress Controller image from the F5 Docker registry." +weight: 150 +doctypes: [""] +toc: true +--- + +{{}} + +## Overview + +Follow the steps in this document to pull the NGINX Plus Ingress Controller image from the F5 Docker registry into your Kubernetes cluster using your JWT token. + +{{}} +An NGINX Plus subscription certificate and key will not work with the F5 Docker registry. + +For NGINX Ingress Controller, you must have an NGINX Ingress Controller subscription -- download the NGINX Plus Ingress Controller (per instance) JWT access token from [MyF5](https://my.f5.com). + +To list the available image tags using the Docker registry API, you will also need to download the NGINX Plus Ingress Controller (per instance) certificate (`nginx-repo.crt`) and the key (`nginx-repo.key`) from [MyF5](https://my.f5.com).{{}} + +{{< note >}} +You can also get the image using alternative methods: + +* You can use Docker to pull an NGINX Ingress Controller image with NGINX Plus and push it to your private registry by following the ["Pulling the Ingress Controller Image"]({{< relref "installation/nic-images/pulling-ingress-controller-image" >}}) documentation. +* You can build an NGINX Ingress Controller image by following the ["Information on how to build an Ingress Controller image"]({{< relref "installation/building-nginx-ingress-controller" >}}) documentation. + +If you would like to use an NGINX Ingress Controller image using NGINX open source, we provide the image through [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/). +{{< /note >}} + +## Before You Begin + +You will need the following information from [MyF5](https://my.f5.com) for these steps: + +- A JWT Access Token (per instance) for NGINX Ingress Controller from an active NGINX Ingress Controller subscription. +- The certificate (`nginx-repo.crt`) and key (`nginx-repo.key`) for each NGINX Ingress Controller instance, used to list the available image tags from the Docker registry API. + +## Prepare NGINX Ingress Controller + +1. Choose your desired [NGINX Ingress Controller Image](https://docs.nginx.com/nginx-ingress-controller/technical-specifications/#images-with-nginx-plus). +1. Log into the [MyF5 Portal](https://my.f5.com/), navigate to your subscription details, and download the relevant .cert, .key and .JWT files. +1. Create a Kubernetes secret using the JWT token. You should use `cat` to view the contents of the JWT token and store the output for use in later steps. +1. Ensure there are no additional characters or extra whitespace that might have been accidentally added. This will break authorization and prevent the NGINX Ingress Controller image from being downloaded. +1. Modify your deployment (manifest or Helm) to use the Kubernetes secret created in step 3. +1. Deploy NGINX Ingress Controller into your Kubernetes cluster and verify that the installation has been successful. + +## Using the JWT token in a Docker Config Secret + +1. Create a Kubernetes `docker-registry` secret type on the cluster, using the JWT token as the username and `none` for password (as the password is not used). The name of the docker server is `private-registry.nginx.com`. + + ```shell + kubectl create secret docker-registry regcred --docker-server=private-registry.nginx.com --docker-username= --docker-password=none [-n nginx-ingress] + ``` + + It is important that the `--docker-username=` contains the contents of the token and is not pointing to the token itself. Ensure that when you copy the contents of the JWT token, there are no additional characters or extra whitespaces. This can invalidate the token and cause 401 errors when trying to authenticate to the registry. + +1. Confirm the details of the created secret by running: + + ```shell + kubectl get secret regcred --output=yaml + ``` + +1. You can now use the newly created Kubernetes secret in Helm and manifest deployments. + +--- + +## Manifest Deployment + +The page ["Installation with Manifests"](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/) explains how to install NGINX Ingress Controller using manifests. The following snippet is an example of a deployment: + +```yaml +spec: + serviceAccountName: nginx-ingress + imagePullSecrets: + - name: regcred + automountServiceAccountToken: true + securityContext: + seccompProfile: + type: RuntimeDefault + containers: + - image: private-registry.nginx.com/nginx-ic/nginx-plus-ingress:3.2.0 + imagePullPolicy: IfNotPresent + name: nginx-plus-ingress +``` + +The `imagePullSecrets` and `containers.image` lines represent the Kubernetes secret, as well as the registry and version of the NGINX Ingress Controller we are going to deploy. + +--- + +## Helm Deployment + +If you are using Helm for deployment, there are two main methods: using *sources* or *charts*. + +### Helm Source + +The [Installation with Helm ]({{< relref "installation/installing-nic/installation-with-helm.md#managing-the-chart-via-sources" >}}) documentation has a section describing how to use sources: these are the unique steps for Docker secrets using JWT tokens. + +1. Clone the NGINX [`kubernetes-ingress` repository](https://github.com/nginxinc/kubernetes-ingress). +1. Navigate to the `deployments/helm-chart` folder of your local clone. +1. Open the `values.yaml` file in an editor. + + You must change a few lines NGINX Ingress Controller with NGINX Plus to be deployed. + + 1. Change the `nginxplus` argument to `true`. + 1. Change the `repository` argument to the NGINX Ingress Controller image you intend to use. + 1. Add an argument to `imagePullSecretName` to allow Docker to pull the image from the private registry. + + The following code block shows snippets of the parameters you will need to change, and an example of their contents: + + ```yaml + ## Deploys the Ingress Controller for NGINX Plus + nginxplus: true + ## Truncated fields + ## ... + ## ... + image: + ## The image repository for the desired NGINX Ingress Controller image + repository: private-registry.nginx.com/nginx-ic/nginx-plus-ingress + + ## The version tag + tag: 3.2.0 + + serviceAccount: + ## The annotations of the service account of the Ingress Controller pods. + annotations: {} + + ## Truncated fields + ## ... + ## ... + + ## The name of the secret containing docker registry credentials. + ## Secret must exist in the same namespace as the helm release. + imagePullSecretName: regcred + ``` + +With the modified `values.yaml` file, you can now use Helm to install NGINX Ingress Controller, for example: + +```shell +helm install nicdev01 -n nginx-ingress --create-namespace -f values.yaml . +``` + +The above command will install NGINX Ingress Controller in the `nginx-ingress` namespace. + +If the namespace does not exist, `--create-namespace` will create it. Using `-f values.yaml` tells Helm to use the `values.yaml` file that you modified earlier with the settings you want to apply for your NGINX Ingress Controller deployment. + + +### Helm Chart + +If you want to install NGINX Ingress Controller using the charts method, the following is an example of using the command line to pass the required arguments using the `set` parameter. + +```shell +helm install my-release -n nginx-ingress oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.18.0 --set controller.image.repository=private-registry.nginx.com/nginx-ic/nginx-plus-ingress --set controller.image.tag=3.2.0 --set controller.nginxplus=true --set controller.serviceAccount.imagePullSecretName=regcred +``` +You can also use the certificate and key from the MyF5 portal and the Docker registry API to list the available image tags for the repositories, for example: + +```shell + $ curl https://private-registry.nginx.com/v2/nginx-ic/nginx-plus-ingress/tags/list --key --cert | jq + + { + "name": "nginx-ic/nginx-plus-ingress", + "tags": [ + "3.2.1-alpine", + "3.2.1-ubi", + "3.2.1" + ] + } + + $ curl --key --cert | jq + { + "name": "nginx-ic-nap/nginx-plus-ingress", + "tags": [ + "3.2.1-ubi", + "3.2.1" + ] + } + + $ curl --key --cert | jq + { + "name": "nginx-ic-dos/nginx-plus-ingress", + "tags": [ + "3.2.1-ubi", + "3.2.1" + ] + } +``` + +--- + +## Pulling an Image for Local Use + +If you need to pull the image for local use to then push to a different container registry, use this command: + +```shell +docker login private-registry.nginx.com --username= --password=none +``` + +Replace the contents of `` with the contents of the JWT token itself. +Once you have successfully pulled the image, you can then tag it as needed. diff --git a/docs/content/installation/pulling-ingress-controller-image.md b/docs/content/installation/pulling-ingress-controller-image.md deleted file mode 100644 index 62f57b79000..00000000000 --- a/docs/content/installation/pulling-ingress-controller-image.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -title: Pulling the NGINX Ingress Controller Image -description: "This document explains how to pull a F5 NGINX Plus Ingress Controller image from the F5 Docker registry." -weight: 1700 -doctypes: [""] -toc: true -docs: "DOCS-605" ---- - -This document explains how to pull an NGINX Plus Ingress Controller image from the F5 Docker registry using your NGINX Ingress Controller subscription certificate and key. **Please note that an NGINX Plus subscription certificate and key will not work with the F5 Docker registry.** You can also get the image using the following alternate methods: - -- [Install using a JWT token in a Docker Config Secret]({{< relref "using-the-jwt-token-docker-secret" >}}) -- [Build the Ingress Controller image]({{< relref "building-ingress-controller-image" >}}) using the source code from the GitHub repository and your NGINX Plus subscription certificate and key. -- For NGINX Ingress Controller based on NGINX OSS, you can pull the [nginx/nginx-ingress image](https://hub.docker.com/r/nginx/nginx-ingress/) from DockerHub. - -## Prerequisites - -Before you can pull the image, make sure that the following software is installed on your machine: - -- [Docker](https://www.docker.com/products/docker) v18.09+ -- For NGINX Ingress Controller, you must have the NGINX Ingress Controller subscription -- download the NGINX Plus Ingress Controller (per instance) certificate (`nginx-repo.crt`) and the key (`nginx-repo.key`) from [MyF5](https://my.f5.com). - -## Pulling the Image using Docker and Pushing It to the Private Registry - -1. First, configure the Docker environment to use certificate-based client-server authentication with the F5 Container registry - `private-registry.nginx.com`. - To do so in a Linux based environment, create a `private-registry.nginx.com` directory under `/etc/docker/certs.d` and create a certificate `client.cert` (using `nginx-repo.crt` - please note that the certificate MUST have the `.cert` suffix, not `.crt`) and a key `client.key` (using `nginx-repo.key`). See the [Docker Engine Security documentation](https://docs.docker.com/engine/security/certificates/) for more details. - - ```console - mkdir -p /etc/docker/certs.d/private-registry.nginx.com - cp nginx-repo.crt /etc/docker/certs.d/private-registry.nginx.com/client.cert - cp nginx-repo.key /etc/docker/certs.d/private-registry.nginx.com/client.key - ``` - - > **Note**: The preceding example is operating-system specific and is for illustrative purposes only. You should consult your operating system documentation for creating an os-provided bundled certificate chain. For example, to configure this for Docker Desktop for Mac or Docker Desktop for Windows, see the [Docker for Mac documentation](https://docs.docker.com/docker-for-mac/#add-client-certificates) or [Docker for Windows documentation](https://docs.docker.com/docker-for-windows/#how-do-i-add-client-certificates) for more details. - -2. Use Docker to pull the required image from `private-registry.nginx.com`. Choose the image from the available images listed in the [tech specs guide]({{< relref "technical-specifications#images-with-nginx-plus" >}}). - For NGINX Plus Ingress Controller, pull from `private-registry.nginx.com/nginx-ic/nginx-plus-ingress`. For example: - - ```console - docker pull private-registry.nginx.com/nginx-ic/nginx-plus-ingress:3.3.2 - ``` - - For NGINX Plus Ingress Controller with App Protect WAF, pull from `private-registry.nginx.com/nginx-ic-nap/nginx-plus-ingress`. For example: - - ```console - docker pull private-registry.nginx.com/nginx-ic-nap/nginx-plus-ingress:3.3.2 - ``` - - For NGINX Plus Ingress Controller with App Protect DoS, pull from `private-registry.nginx.com/nginx-ic-dos/nginx-plus-ingress`. For example: - - ```console - docker pull private-registry.nginx.com/nginx-ic-dos/nginx-plus-ingress:3.3.2 - ``` - - To list the available image tags for the repositories, you can also use the Docker registry API. For example: - - ```json - $ curl https://private-registry.nginx.com/v2/nginx-ic/nginx-plus-ingress/tags/list --key --cert | jq - { - "name": "nginx-ic/nginx-plus-ingress", - "tags": [ - "3.3.2-alpine", - "3.3.2-ubi", - "3.3.2" - ] - } - - $ curl https://private-registry.nginx.com/v2/nginx-ic-nap/nginx-plus-ingress/tags/list --key --cert | jq - { - "name": "nginx-ic-nap/nginx-plus-ingress", - "tags": [ - "3.3.2-ubi", - "3.3.2" - ] - } - - $ curl https://private-registry.nginx.com/v2/nginx-ic-dos/nginx-plus-ingress/tags/list --key --cert | jq - { - "name": "nginx-ic-dos/nginx-plus-ingress", - "tags": [ - "3.3.2-ubi", - "3.3.2" - ] - } - ``` - -3. Tag and push the image to your private registry. - - - Make sure to run the `docker login ` command first to log in to the registry. - - Replace `` in the examples below with the correct path to your private Docker registry. - - ```console - docker tag private-registry.nginx.com/nginx-ic/nginx-plus-ingress:3.3.2 /nginx-ic/nginx-plus-ingress:3.3.2 - docker push /nginx-ic/nginx-plus-ingress:3.3.2 - ``` - - or for NGINX App Protect WAF enabled image - - ```console - docker tag private-registry.nginx.com/nginx-ic-nap/nginx-plus-ingress:3.3.2 /nginx-ic-nap/nginx-plus-ingress:3.3.2 - docker push /nginx-ic-nap/nginx-plus-ingress:3.3.2 - ``` - - or for NGINX App Protect DoS enabled image - - ```console - docker tag private-registry.nginx.com/nginx-ic-dos/nginx-plus-ingress:3.3.2 /nginx-ic-dos/nginx-plus-ingress:3.3.2 - docker push /nginx-ic-dos/nginx-plus-ingress:3.3.2 - ``` diff --git a/docs/content/installation/running-multiple-ingress-controllers.md b/docs/content/installation/running-multiple-ingress-controllers.md index 7ba52b64399..16fb919015a 100644 --- a/docs/content/installation/running-multiple-ingress-controllers.md +++ b/docs/content/installation/running-multiple-ingress-controllers.md @@ -1,8 +1,8 @@ --- -title: Running Multiple Ingress Controllers +title: Running Multiple NGINX Ingress Controllers -description: "This document explains how to run multiple NGINX Ingress Controllers." -weight: 2100 +description: "Learn how to run multiple NGINX Ingress Controllers." +weight: 400 doctypes: [""] aliases: - /multiple-ingress-controllers/ @@ -10,50 +10,45 @@ toc: true docs: "DOCS-606" --- - This document explains the following topics: - Ingress class concept. - How to run NGINX Ingress Controller in the same cluster with another Ingress Controller, such as an Ingress Controller for a cloud HTTP load balancer, and prevent any conflicts between the Ingress Controllers. - How to run multiple NGINX Ingress Controllers. -**Note**: In this document we refer to Ingress, VirtualServer, VirtualServerRoute, and TransportServer resources as configuration resources. +{{< note >}}In this document we refer to the [Ingress]({{< relref "/configuration/ingress-resources/basic-configuration.md" >}}), [VirtualServer]({{< relref "/configuration/virtualserver-and-virtualserverroute-resources.md#virtualserver-specification" >}}), [VirtualServerRoute]({{< relref "/configuration/virtualserver-and-virtualserverroute-resources.md#virtualserverroute-specification" >}}), and [TransportServer]({{< relref "/configuration/transportserver-resource.md" >}}) resources as "configuration resources".{{< /note >}} -## Ingress Class +## Ingress class -The smooth coexistence of multiple Ingress Controllers in one cluster is provided by the Ingress class concept, which mandates the following: +Thanks to the Ingress class concept, multiple Ingress Controllers can coexist in one cluster. The Ingress class has the following characteristics: - Every Ingress Controller must only handle Ingress resources for its particular class. -- Ingress resources should have the `ingressClassName` field set to the value, which corresponds to the class of the Ingress Controller the user wants to use. -- VirtualServer, VirtualServerRoute, Policy and TransportServer resources should have the `ingressClassName` field set to the value, which corresponds to the class of the Ingress Controller the user wants to use. - -### Configuring Ingress Class +- Ingress resources need to have the `ingressClassName` field set to the value of the class of the Ingress Controller the user wants to use. +- VirtualServer, VirtualServerRoute, Policy, and TransportServer resources need to have the `ingressClassName` field set to the value of the class of the Ingress Controller the user wants to use. -The default Ingress class of NGINX Ingress Controller is `nginx`, which means that it only handles configuration resources with the `class` set to `nginx`. You can customize the class through the `-ingress-class` command-line argument. +### Configuring Ingress class -**Notes**: +The default Ingress class of NGINX Ingress Controller is `nginx`, which means that it only handles configuration resources with the Ingress class set to `nginx`. You can customize the class through the `-ingress-class` command-line argument. -- If the class is not set in an Ingress resource, Kubernetes will set it to the class of the default Ingress Controller. To make the Ingress Controller the default one, the `ingressclass.kubernetes.io/is-default-class` must be set on the IngressClass resource. See Step 3 *Create an IngressClass resource* of the [Create Common Resources](/nginx-ingress-controller/installation/installation-with-manifests/#2-create-common-resources) section. -- For VirtualServer, VirtualServerRoute, Policy and TransportServer resources the Ingress Controller will always handle resources with an empty class. +{{< note >}}- If the class of an Ingress resource is not set, Kubernetes will set it to the class of the default Ingress Controller. To make the Ingress Controller the default one, the `ingressclass.kubernetes.io/is-default-class` property must be set on the IngressClass resource. To learn more, see Step 3 *Create an IngressClass resource* of the [Create Common Resources]({{< relref "installation/installing-nic/installation-with-manifests.md#create-common-resources" >}}) section. +- For VirtualServer, VirtualServerRoute, Policy and TransportServer resources, NGINX Ingress Controller will always handle resources with an empty class.{{< /note >}} -## Running NGINX Ingress Controller and Another Ingress Controller +## Running NGINX Ingress Controller and another Ingress Controller -It is possible to run NGINX Ingress Controller and an Ingress Controller for another load balancer in the same cluster. This is often the case if you create your cluster through a cloud provider managed Kubernetes service that by default might include the Ingress Controller for the HTTP load balancer of the cloud provider, and you want to use NGINX Ingress Controller. +It is possible to run NGINX Ingress Controller and an Ingress Controller for another load balancer in the same cluster. This is often the case if you create your cluster through a cloud provider's managed Kubernetes service that by default might include the Ingress Controller for the HTTP load balancer of the cloud provider, and you want to use NGINX Ingress Controller. -To make sure that NGINX Ingress Controller handles particular configuration resources, update those resources with the `class` set to `nginx` or the value that you configured. +To make sure that NGINX Ingress Controller handles specific configuration resources, update those resources with the class set to `nginx` (or the Ingress class value that you configured in NGINX Ingress Controller). -## Running Multiple NGINX Ingress Controllers +## Running multiple NGINX Ingress Controllers When running NGINX Ingress Controller, you have the following options with regards to which configuration resources it handles: -- **Cluster-wide Ingress Controller (default)**. The Ingress Controller handles configuration resources created in any namespace of the cluster. As NGINX is a high-performance load balancer capable of serving many applications at the same time, this option is used by default in our installation manifests and Helm chart. -- **Defined-namespace Ingress Controller**. You can configure the Ingress Controller to handle configuration resources only from particular namespaces, which is controlled through the `-watch-namespace` command-line argument. This can be useful if you want to use different NGINX Ingress Controllers for different applications, both in terms of isolation and/or operation. -- **Ingress Controller for Specific Ingress Class**. This option works in conjunction with either of the options above. You can further customize which configuration resources are handled by the Ingress Controller by configuring the class of the Ingress Controller and using that class in your configuration resources. See the section [Configuring Ingress Class](#configuring-ingress-class). - -Considering the options above, you can run multiple NGINX Ingress Controllers, each handling a different set of configuration resources. +- Cluster-wide Ingress Controller (default): NGINX Ingress Controller handles configuration resources created in any namespace of the cluster. As NGINX is a high-performance load balancer capable of serving many applications at the same time, this option is used by default in our installation manifests and Helm chart. +- Defined-namespace Ingress Controller: You can configure the Ingress Controller to handle configuration resources only from particular namespaces, which is controlled through the `-watch-namespace` command-line argument. This can be useful if you want to use different NGINX Ingress Controllers for different applications, both in terms of isolation and/or operation. +- Ingress Controller for Specific Ingress Class: This option works in conjunction with either of the options above. You can further customize which configuration resources are handled by the Ingress Controller by configuring the class of the Ingress Controller and using that class in your configuration resources. See the section [Configuring Ingress Class](#configuring-ingress-class). -## See Also +Using the options above you can run multiple NGINX Ingress Controllers, each handling a different set of configuration resources. -- [Command-line arguments](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments) +{{< see-also >}}[Command-line arguments]({{< relref "configuration/global-configuration/command-line-arguments" >}}){{< /see-also >}} -**Note**: all mentioned command-line arguments are also available as the parameters in the [Helm chart](/nginx-ingress-controller/installation/installation-with-helm). +{{< note >}}All the mentioned command-line arguments are also available as parameters in the [Helm chart]({{< relref "installation/installing-nic/installation-with-helm" >}}).{{< /note >}} diff --git a/docs/content/installation/using-aws-marketplace-image.md b/docs/content/installation/using-aws-marketplace-image.md deleted file mode 100644 index f98ca63a93f..00000000000 --- a/docs/content/installation/using-aws-marketplace-image.md +++ /dev/null @@ -1,86 +0,0 @@ ---- -title: Using the AWS Marketplace Ingress Controller Image -description: "This document will walk you through the steps needed to use the NGINX Ingress Controller through the AWS Marketplace." -weight: 2300 -doctypes: [""] -toc: true -docs: "DOCS-607" ---- - -This document will walk you through the steps needed to use the NGINX Ingress Controller through the AWS Marketplace. There are additional steps that must be followed in order for the AWS Marketplace NGINX Ingress Controller to work properly. - -> **IMPORTANT**: This document uses EKS version 1.19. EKS versions < 1.19 require additional security settings within the NGINX Pod to work properly with marketplace images. -> This document discusses using eksctl to perform necessary steps to enable the Kubernetes cluster access to deploy NGINX Ingress Controller from the Marketplace. Please make sure you are running a newer version of eksctl and AWS cli. - -> **NOTE**: NGINX Ingress Controller from the Marketplace does NOT work in AWS Region US-West-1. - -## Instructions - -Instructions for using AWS Marketplace: - -1. Ensure you have a working AWS EKS cluster. If you do not have a EKS cluster, you can create one using either the AWS console, or using the AWS tool eksctl. See [this guide](https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html) for details on getting started with EKS using eksctl. - -2. You must create a new IAM role that will be associated with the Service Account created for the NGINX Ingress Controller. This IAM role will have a specific IAM policy that allows you to monitor the usage of the AWS NGINX Ingress Controller image. This is a required step. If it is omitted, AWS Marketplace NGINX Ingress Controller will not work properly and NGINX Ingress will not start. Please see [the AWS EKS IAM documentation here](https://docs.aws.amazon.com/eks/latest/userguide/create-service-account-iam-policy-and-role.html) and the documentation detailing the policy required for the [AWS Marketplace here](https://docs.aws.amazon.com/marketplace/latest/userguide/iam-user-policy-for-aws-marketplace-actions.html). - -3. You must associate this IAM role with your service account in your EKS cluster. When you do so, your service account Kubernetes object will have a annotation, showing the link to the IAM role. - -> **NB** You must associate your AWS EKS cluster with an OIDC provider before you can create your IAM Service account! This is required. - -## Step by step instructions using eksctl utility - -This assumes you have an existing EKS cluster up and running. If not, please create one before proceeding. This also assumes the namespace for the NGINX Ingress Controller already exists. - -1. Associate your EKS cluster with a “OIDC IAM provider” (replace `--cluster ` and `--region ` with the values of your environment). - -``` -eksctl utils associate-iam-oidc-provider --region=eu-west-1 --cluster=json-eu-east1 --approve -``` - -2. Now create your IAM role and service account for your cluster. Substitute `--name ` `--namespace ` and `--region ` with your values. - -``` -eksctl create iamserviceaccount --name nginx-ingress --namespace nginx-ingress --cluster json-test01 --region us-east-2 --attach-policy-arn arn:aws:iam::aws:policy/AWSMarketplaceMeteringRegisterUsage --approve - ``` - -This will create the IAM role with the required policy attached, create the service account if it doesn't already exist, and add the annotations needed for your AWS cluster. See [the documentation here](https://docs.aws.amazon.com/eks/latest/userguide/create-service-account-iam-policy-and-role.html). Since eksctl is creating it for you, you do not need to apply any service account yaml files for your NGINX Ingress Controller deployments. - -``` -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - EKS.amazonaws.com/role-arn: arn:aws:iam::001234567890:role/eksctl-json-us-west2-addon-iamserviceaccount-Role1-IJJ6CF9Y8IPY - labels: - app.kubernetes.io/managed-by: eksctl - name: nginx-ingress - namespace: nginx-ingress -secrets: -- name: nginx-ingress-token-zm728 -``` - -Make sure you match the name you are creating for the service account, to the account that will be in the `rbac.yaml` file for manifest deployment. - -Sample output from the `rbac.yaml` file, matching the IAM service account that was created above: - -``` -kind: ClusterRoleBinding - apiVersion: rbac.authorization.k8s.io/v1 - metadata: - name: nginx-ingress - subjects: - - kind: ServiceAccount - name: nginx-ingress - namespace: nginx-ingress - roleRef: - kind: ClusterRole - name: nginx-ingress - apiGroup: rbac.authorization.k8s.io -``` - -3. Log into the AWS ECR registry that is specified in the instructions from the [AWS Marketplace portal](https://aws.amazon.com/marketplace/pp/prodview-fx3faxl7zqeau?sr=0-1&ref_=beagle&applicationId=AWSMPContessa). - -{{< img title="ECR Pull Instructions for the NGINX Ingress Controller" src="./img/ecr-pull-instructions.png" >}} - -**Note:** AWS Labs also provides a credential helper - see [their GitHub repo](https://github.com/awslabs/amazon-ecr-credential-helper) for instructions on how to setup and configure. - -4. Update the image in the `nginx-plus-ingress.yaml` manifest. diff --git a/docs/content/installation/using-gcp-marketplace-package.md b/docs/content/installation/using-gcp-marketplace-package.md deleted file mode 100644 index bf88f2345ef..00000000000 --- a/docs/content/installation/using-gcp-marketplace-package.md +++ /dev/null @@ -1,139 +0,0 @@ ---- -title: Using GCP Marketplace Ingress Controller -description: "This document will walk you through the steps needed to deploy the NGINX Ingress Controller through the GCP Marketplace." -weight: 2400 -doctypes: [""] -toc: true -docs: "DOCS-000" ---- - -This document will walk you through the steps needed to deploy and configure the NGINX Ingress Controller through the GCP Marketplace. - -## Product overview - -The F5 NGINX Ingress Controller is an implementation of a Kubernetes Ingress Controller for NGINX and NGINX Plus. - -Basic features include: - -- Host-based routing. For example, routing requests with the host header foo.example.com to one group of services and the host header bar.example.com to another group. -- Path-based routing. For example, routing requests with the URI that starts with /serviceA to service A and requests with the URI that starts with /serviceB to service B. -- TLS/SSL termination for each hostname, such as foo.example.com. - -## One-time setup - -To quickly get the NGINX Ingress Controller up and running, follow our [Installation with Manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/) guide. -This details how to locally build an image of the NGINX Ingress Controller as well as the necessary CustomResourceDefinitions (CRDs) required. - -## Installation - -### Install NGINX Ingress Controller to an existing GKE cluster - -1. Deploy NGINX Ingress Controller from GUI - - Open [Google Cloud Console](https://console.cloud.google.com/) and navigate to **Kubernetes Engine > Applications** - -2. Click on **DEPLOY FROM MARKETPLACE**, and search for *NGINX Ingress Controller* - - - > **NOTE**: Please verify that you select a *Premium Edition* controller image that has been published by *NGINX, Inc.*, and not a third-party. - - Choose the appropriate *NGINX Ingress Controller* image, and click **CONFIGURE** - -3. Install to an existing cluster - - > **NOTE**: Click on **OR SELECT AN EXISTING CLUSTER** if you see the **CREATE NEW CLUSTER** button. - - {{< img title="Install to existing GKE cluster" src="./img/gke-existing-cluster.png" >}} - - Choose an **Existing Kubernetes Cluster** to target from the options provided. - - The *default* namespace will be selected automatically, but you can choose to **Create a namespace** and enter the **New namespace name**. - - The **App instance name** will be used as a prefix for all resources created by the deployment, and must be unique within the selected namespace. A default value will be generated and can be used as-is, or changed. - - Recommended options for NGINX Ingress Controller will be pre-selected, but can be adjusted as necessary. - - Click **DEPLOY** to launch NGINX Ingress Controller installation process. - - {{< img title="Install to existing GKE cluster" src="./img/gke-ingress-controller-application.png" >}} - - You can find the NGINX Ingress Controller *application* by navigating back to **Kubernetes Engine > Applications** - -### Install NGINX Ingress Controller to a new GKE cluster - -As an alternative to using an existing GKE cluster, GCP Marketplace can create a small zonal cluster for you during the installation process. Please note that you will need to resize the cluster to provide enough vCPU for NGINX Ingress Controller and your other applications. - -1. Find NGINX Ingress Controller on the marketplace - - Open [Google Cloud Console](https://console.cloud.google.com/) and navigate to **Marketplace**, then search for *NGINX Ingress Controller* - - - > **NOTE**: Please verify that you select a *Premium Edition* controller image that has been published by *NGINX, Inc.*, and not a third-party. - - Choose the appropriate *NGINX Ingress Controller* image, and click **CONFIGURE** - -2. Configure the new GKE cluster - - Choose the **zone**, **network**, and **subnetwork** that is appropriate for your new cluster from the options provided. - - {{< img title="Create a new GKE cluster" src="./img/gke-create-cluster.png" >}} - - Click **CREATE NEW CLUSTER**. - - {{< img title="New cluster creation in progress" src="./img/gke-creating-cluster.png" >}} - - After a short delay the cluster will be ready for step 3. - -3. Complete the NGINX Ingress Controller deployment options - - {{< img title="Complete new GKE cluster installation" src="./img/gke-install-to-new-cluster.png" >}} - - The *default* namespace will be selected automatically, but you can choose to **Create a namespace** and enter the **New namespace name**. - - The **App instance name** will be used as a prefix for all resources created by the deployment, and must be unique within the selected namespace. A default value will be generated and can be used as-is, or changed. - - Recommended options for NGINX Ingress Controller will be pre-selected, but can be adjusted as necessary. - - Click **DEPLOY** to launch NGINX Ingress Controller installation process. - - {{< img title="Install to existing GKE cluster" src="./img/gke-ingress-controller-application.png" >}} - - You can find the NGINX Ingress Controller *application* by navigating back to **Kubernetes Engine > Applications** - -## Configuration - -The GCP Marketplace will deploy the NGINX Ingress Controller with a default configuration and an empty *ConfigMap*. These resources will all be named `-nginx-ingress`, where `` matches the value you provided during the [installation](#installation) step. - -For example, if NGINX Ingress Controller was deployed to namespace `nginx-ingress` and with an **App instance name** of `nginx-ingress-plus` (as used in the examples above), the ConfigMap can be viewed with `kubectl`: - -``` -$ kubectl get configmap -n nginx-ingress nginx-ingress-plus-nginx-ingress -o yaml -apiVersion: v1 -kind: ConfigMap -metadata: - annotations: - kubectl.kubernetes.io/last-applied-configuration: | - {"apiVersion":"v1","data":null,"kind":"ConfigMap","metadata":{"annotations":{},"labels":{"app.kubernetes.io/instance":"nginx-ingress-plus","app.kubernetes.io/managed-by":"Helm","app.kubernetes.io/name":"nginx-ingress-plus-nginx-ingress","helm.sh/chart":"nginx-ingress-0.16.2"},"name":"nginx-ingress-plus-nginx-ingress","namespace":"nginx-ingress","ownerReferences":[{"apiVersion":"app.k8s.io/v1beta1","blockOwnerDeletion":true,"kind":"Application","name":"nginx-ingress-plus","uid":"5cbbebd8-df13-4001-bd65-9467405d9a9d"}]}} - creationTimestamp: "2022-08-25T01:03:10Z" - labels: - app.kubernetes.io/instance: nginx-ingress-plus - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: nginx-ingress-plus-nginx-ingress - helm.sh/chart: nginx-ingress-0.16.2 - name: nginx-ingress-plus-nginx-ingress - namespace: nginx-ingress - ownerReferences: - - apiVersion: app.k8s.io/v1beta1 - blockOwnerDeletion: true - kind: Application - name: nginx-ingress-plus - uid: 5cbbebd8-df13-4001-bd65-9467405d9a9d - resourceVersion: "147519" - uid: 3fa33891-7a30-4004-91bd-bd5d652e34a9 -``` - -See the [Configuration](https://docs.nginx.com/nginx-ingress-controller/configuration/) documentation to modify the resources. - -## Basic Usage - -To setup a basic application that uses the NGINX Ingress Controller, see our [basic configuration](https://github.com/nginxinc/kubernetes-ingress/tree/main/examples/custom-resources/basic-configuration) example page. diff --git a/docs/content/installation/using-the-jwt-token-docker-secret.md b/docs/content/installation/using-the-jwt-token-docker-secret.md deleted file mode 100644 index 8ccd0bec631..00000000000 --- a/docs/content/installation/using-the-jwt-token-docker-secret.md +++ /dev/null @@ -1,189 +0,0 @@ ---- -title: Using NGINX Ingress Controller Plus JWT token in a Docker Config Secret -description: "This document explains how to use the NGINX Plus Ingress Controller image from the F5 Docker registry in your Kubernetes cluster by using an NGINX Ingress Controller subscription JWT token." -weight: 1600 -doctypes: [""] -toc: true ---- - -## Overview - -This document explains how to pull the NGINX Plus Ingress Controller image from the F5 Docker registry into your Kubernetes cluster using your JWT token. - -{{}} -An NGINX Plus subscription certificate and key will not work with the F5 Docker registry. -For NGINX Ingress Controller, you must have the NGINX Ingress Controller subscription -- download the NGINX Plus Ingress Controller (per instance) JWT access token from [MyF5](https://my.f5.com). -To list the available image tags using the Docker registry API, you will also need to download the NGINX Plus Ingress Controller (per instance) certificate (`nginx-repo.crt`) and the key (`nginx-repo.key`) from [MyF5](https://my.f5.com). -{{}} - -You can also get the image using alternative methods: - -- You can use Docker to pull an NGINX Ingress Controller image with NGINX Plus and push it to your private registry by following the ["Pulling the Ingress Controller Image"](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) documentation. -- You can also build an NGINX Ingress Controller image by following the ["Information on how to build an Ingress Controller image"](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/) documentation. - -If you would like an NGINX Ingress Controller image using NGINX open source, we provide the image through [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/). - -## Before You Begin - -You will need the following information from [MyF5](https://my.f5.com) for these steps: - -- A JWT Access Token (Per instance) for NGINX Ingress Controller from an active NGINX Ingress Controller subscription. -- The certificate (`nginx-repo.crt`) and key (`nginx-repo.key`) for each NGINX Ingress Controller instance, used to list the available image tags from the Docker registry API. - -## Prepare NGINX Ingress Controller - -1. Choose your desired [NGINX Ingress Controller Image](https://docs.nginx.com/nginx-ingress-controller/technical-specifications/#images-with-nginx-plus). -1. Log into the [MyF5 Portal](https://myf5.com/), navigate to your subscription details, and download the relevant .cert, .key and .JWT files. -1. Create a Kubernetes secret using the JWT token. You should use `cat` to view the contents of the JWT token and store the output for use in later steps. -1. Ensure there are no additional characters or extra whitespace that might have been accidentally added. This will break authorization and prevent the NGINX Ingress Controller image from being downloaded. -1. Modify your deployment (manifest or helm) to use the Kubernetes secret created in step three. -1. Deploy NGINX Ingress Controller into your Kubernetes cluster and verify successful installation. - -## Using the JWT token in a Docker Config Secret - -1. Create a Kubernetes `docker-registry` secret type on the cluster, using the JWT token as the username and `none` for password (Password is unused). The name of the docker server is `private-registry.nginx.com`. - - ```shell - kubectl create secret docker-registry regcred --docker-server=private-registry.nginx.com --docker-username= --docker-password=none [-n nginx-ingress] - ``` - - It is important that the `--docker-username=` contains the contents of the token and is not pointing to the token itself. Ensure that when you copy the contents of the JWT token, there are no additional characters or extra whitespaces. This can invalidate the token and cause 401 errors when trying to authenticate to the registry. - -1. Confirm the details of the created secret by running: - - ```shell - kubectl get secret regcred --output=yaml - ``` - -1. You can now use the newly created Kubernetes secret in `helm` and `manifest` deployments. - -## Manifest Deployment - -The page ["Installation with Manifests"](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/) explains how to install NGINX Ingress Controller using manifests. The following snippet is an example of a deployment: - -```yaml -spec: - serviceAccountName: nginx-ingress - imagePullSecrets: - - name: regcred - automountServiceAccountToken: true - securityContext: - seccompProfile: - type: RuntimeDefault - containers: - - image: private-registry.nginx.com/nginx-ic/nginx-plus-ingress:3.3.0 - imagePullPolicy: IfNotPresent - name: nginx-plus-ingress -``` - -The `imagePullSecrets` and `containers.image` lines represent the Kubernetes secret, as well as the registry and version of the NGINX Ingress Controller we are going to deploy. - -## Helm Deployment - -If you are using `helm` for deployment, there are two main methods: using *sources* or *charts*. - -### Helm Source - -The [Helm installation page for NGINX Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-helm/#managing-the-chart-via-sources) has a section describing how to use sources: these are the unique steps for Docker secrets using JWT tokens. - -1. Clone the NGINX [`kubernetes-ingress` repository](https://github.com/nginxinc/kubernetes-ingress). -1. Navigate to the `deployments/helm-chart` folder of your local clone. -1. Open the `values.yaml` file in an editor. - -You must change a few lines NGINX Ingress Controller with NGINX Plus to be deployed. - -1. Change the `nginxplus` argument to `true`. -1. Change the `repository` argument to the NGINX Ingress Controller image you intend to use. -1. Add an argument to `imagePullSecretName` to allow Docker to pull the image from the private registry. - -The following code block shows snippets of the parameters you will need to change, and an example of their contents: - -```yaml -## Deploys the Ingress Controller for NGINX Plus -nginxplus: true -## Truncated fields -## ... -## ... -image: - ## The image repository for the desired NGINX Ingress Controller image - repository: private-registry.nginx.com/nginx-ic/nginx-plus-ingress - - ## The version tag - tag: 3.3.0 - - serviceAccount: - ## The annotations of the service account of the Ingress Controller pods. - annotations: {} - - ## Truncated fields - ## ... - ## ... - - ## The name of the secret containing docker registry credentials. - ## Secret must exist in the same namespace as the helm release. - imagePullSecretName: regcred -``` - -With `values.yaml` modified, you can now use Helm to install NGINX Ingress Controller, such as in the following example: - -```shell -helm install nicdev01 -n nginx-ingress --create-namespace -f values.yaml . -``` - -The above command will install NGINX Ingress Controller in the `nginx-ingress` namespace. - -If the namespace does not exist, `--create-namespace` will create it. Using `-f values.yaml` tells `helm` to use the `values.yaml` file that you modified earlier with the settings you want to apply for your NGINX Ingress Controller deployment. - -### Helm Chart - -If you want to install NGINX Ingress Controller using the charts method, the following is an example of using the command line to pass the required arguments using the `set` parameter. - -```shell -helm install my-release -n nginx-ingress oci://ghcr.io/nginxinc/charts/nginx-ingress --version 1.0.0 --set controller.image.repository=private-registry.nginx.com/nginx-ic/nginx-plus-ingress --set controller.image.tag=3.3.0 --set controller.nginxplus=true --set controller.serviceAccount.imagePullSecretName=regcred -``` - -Checking the validation that the .crts/key and .jwt are able to successfully authenticate to the repo to pull NGINX Ingress controller images: - -You can also use the certificate and key from the MyF5 portal and the Docker registry API to list the available image tags for the repositories, e.g.: - -```shell - $ curl https://private-registry.nginx.com/v2/nginx-ic/nginx-plus-ingress/tags/list --key --cert | jq - - { - "name": "nginx-ic/nginx-plus-ingress", - "tags": [ - "3.3.2-alpine", - "3.3.2-ubi", - "3.3.2" - ] - } - - $ curl https://private-registry.nginx.com/v2/nginx-ic-nap/nginx-plus-ingress/tags/list --key --cert | jq - { - "name": "nginx-ic-nap/nginx-plus-ingress", - "tags": [ - "3.3.2-ubi", - "3.3.2" - ] - } - - $ curl https://private-registry.nginx.com/v2/nginx-ic-dos/nginx-plus-ingress/tags/list --key --cert | jq - { - "name": "nginx-ic-dos/nginx-plus-ingress", - "tags": [ - "3.3.2-ubi", - "3.3.2" - ] - } -``` - -## Pulling an Image for Local Use - -If you need to pull the image for local use to then push to a different container registry, here is the command: - -```shell -docker login private-registry.nginx.com --username= --password=none -``` - -Replace the contents of `` with the contents of the `jwt token` itself. -Once you have successfully pulled the image, you can then tag it as needed. diff --git a/docs/content/intro/how-nginx-ingress-controller-works.md b/docs/content/intro/how-nginx-ingress-controller-works.md deleted file mode 100644 index 2253297027f..00000000000 --- a/docs/content/intro/how-nginx-ingress-controller-works.md +++ /dev/null @@ -1,327 +0,0 @@ ---- -title: How NGINX Ingress Controller Works -description: "This document explains how NGINX Ingress Controller works." -weight: 300 -doctypes: [""] -toc: true -docs: "DOCS-609" ---- - - -This document explains how NGINX Ingress Controller works. The target audience includes the following two main groups: - -- *Operators* who would like to know how the software works and also better understand how it can fail. -- *Developers* who would like to [contribute](https://github.com/nginxinc/kubernetes-ingress/blob/main/CONTRIBUTING.md) to the project. - -We assume that the reader is familiar with core Kubernetes concepts, such as Pod, Deployment, Service, and Endpoints. Additionally, we recommend reading [this blog post](https://www.nginx.com/blog/inside-nginx-how-we-designed-for-performance-scale/) for an overview of the NGINX architecture. - -## What is an Ingress Controller? - -An Ingress Controller is a component in a Kubernetes cluster that configures an HTTP load balancer according to Ingress resources created by the cluster user. - -> If you’d like to read more about the Ingress resource, refer to [the official Kubernetes documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/). - -This document is specific to NGINX Ingress Controller, referred to as *Ingress Controller* or *IC*, which is built upon NGINX and NGINX Plus capabilities - -## The Ingress Controller at a High Level - -Let’s begin with a high-level examination of the Ingress Controller (IC). The following figure depicts an example of how the IC exposes two web applications running in a Kubernetes cluster to clients on the internet: - -{{< img title="IC at a high level" src="./img/ic-high-level.png" >}} - -The figure shows: - -- A *Kubernetes cluster*. -- Cluster users *Admin*, *User A* and *User B*, which use the cluster via the *Kubernetes API*. -- *Clients A* and *Clients B*, which connect to the *Applications A* and *B* deployed by the corresponding users. -- *IC*, [deployed by *Admin*](/nginx-ingress-controller/installation/installation-with-manifests) in a pod in the namespace *nginx-ingress* and configured via the *ConfigMap nginx-ingress*. For simplicity, we depict only one IC pod; however, *Admin* typically deploys at least two pods for redundancy. The *IC* uses the *Kubernetes API* to get the latest Ingress resources created in the cluster and then configures *NGINX* according to those resources. -- *Application A* with two pods deployed in the *namespace A* by *User A*. To expose the application to its clients (*Clients A*) via the host `a.example.com`, *User A* creates *Ingress A*. -- *Application B* with one pod deployed in the *namespace B* by *User B*. To expose the application to its clients (*Clients B*) via the host `b.example.com`, *User B* creates *VirtualServer B*. -- *Public Endpoint*, which fronts the *IC* pod(s). This is typically a TCP load balancer (cloud, software, or hardware) or a combination of such load balancer with a NodePort service. *Clients A* and *B* connect to their applications via the *Public Endpoint*. - -The yellow and purple arrows represent connections related to the client traffic, and the black arrows represent access to the Kubernetes API. - -> For simplicity, many necessary Kubernetes resources like Deployment and Services aren't shown, which Admin and the users also need to create. - -Next, let's explore the IC pod. - -## The Ingress Controller Pod - -The IC pod consists of a single container, which in turn includes the following: - -- *IC process*, which configures NGINX according to Ingress and other resources created in the cluster. -- *NGINX master process*, which controls NGINX worker processes. -- *NGINX worker processes*, which handle the client traffic and load balance the traffic to the backend applications. - -The following is an architectural diagram that shows how those processes interact together and with some external processes/entities: - -{{< img title="IC pod" src="./img/ic-pod.png" >}} - -For brevity, we've omitted the suffix process from the description of the processes. - - - -The numbered list that follows describes each connection with its type in curly brackets: - -1. (HTTP) *Prometheus* fetches the IC and NGINX metrics via an HTTP endpoint that the *IC* exposes. The default is ``:9113/metrics``. **Note**: *Prometheus* is not required by the IC, the endpoint can be turned off. -2. (HTTPS) The *IC* reads the *Kubernetes API* to get the latest versions of the resources in the cluster and writes to the API to update the handled resources' statuses and emit events. -3. (HTTP) *Kubelet* probes the *IC* readiness probe (the default is `:8081/nginx-ready`) to consider the IC pod [ready](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions). -4. (File I/O) When the *IC* starts, it reads the *configuration templates* necessary for config generation from the filesystem. The templates are located in the `/` directory of the container and have the `.tmpl` extension. -5. (File I/O) The *IC* writes logs to its *stdout* and *stderr*, which are collected by the container runtime. -6. (File I/O) The *IC* generates NGINX *configuration* based on the resources created in the cluster (refer to [The Ingress Controller is a Kubernetes Controller](#the-ingress-controller-is-a-kubernetes-controller) section for the list of resources) and writes it on the filesystem in the `/etc/nginx` folder. The configuration files have a `.conf` extension. -7. (File I/O) The *IC* writes *TLS certificates* and *keys* from any [TLS Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets) referenced in the Ingress and other resources to the filesystem. -8. (HTTP) The *IC* fetches the [NGINX metrics](https://nginx.org/en/docs/http/ngx_http_stub_status_module.html#stub_status) via the `unix:/var/lib/nginx/nginx-status.sock` UNIX socket and converts it to Prometheus format used in #1. -9. (HTTP) To consider a configuration reload a success, the *IC* ensures that at least one *NGINX worker* has the new configuration. To do that, the *IC* checks a particular endpoint via the `unix:/var/lib/nginx/nginx-config-version.sock` UNIX socket. -10. (N/A) To start NGINX, the *IC* runs the `nginx` command, which launches the *NGINX master*. -11. (Signal) To reload NGINX, the *IC* runs the `nginx -s reload` command, which validates the configuration and sends the [reload signal](https://nginx.org/en/docs/control.html) to the *NGINX master*. -12. (Signal) To shutdown NGINX, the *IC* executes `nginx -s quit` command, which sends the graceful shutdown signal to the *NGINX master*. -13. (File I/O) The *NGINX master* sends logs to its *stdout* and *stderr*, which are collected by the container runtime. -14. (File I/O) The *NGINX master* reads the *TLS cert and keys* referenced in the configuration when it starts or reloads. -15. (File I/O) The *NGINX master* reads *configuration files* when it starts or during a reload. -16. (Signal) The *NGINX master* controls the [lifecycle of *NGINX workers*](https://nginx.org/en/docs/control.html#reconfiguration) it creates workers with the new configuration and shutdowns workers with the old configuration. -17. (File I/O) An *NGINX worker* writes logs to its *stdout* and *stderr*, which are collected by the container runtime. -18. (UDP) An *NGINX worker* sends the HTTP upstream server response latency logs via the Syslog protocol over the UNIX socket `/var/lib/nginx/nginx-syslog.sock` to the *IC*. In turn, the *IC* analyzes and transforms the logs into Prometheus metrics. -19. (HTTP,HTTPS,TCP,UDP) A *client* sends traffic to and receives traffic from any of the *NGINX workers* on ports 80 and 443 and any additional ports exposed by the [GlobalConfiguration resource](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource). -20. (HTTP,HTTPS,TCP,UDP) An *NGINX worker* sends traffic to and receives traffic from the *backends*. -21. (HTTP) *Admin* can connect to the [NGINX stub_status](http://nginx.org/en/docs/http/ngx_http_stub_status_module.html#stub_status) using port 8080 via an *NGINX worker*. **Note**: By default, NGINX only allows connections from `localhost`. - -### Differences for NGINX Plus - -The preceding diagram depicts the IC with NGINX. The IC also supports NGINX Plus with the following important differences: - -- To configure NGINX Plus, Ingress Controller uses [configuration reloads](#reloading-nginx) and the [NGINX Plus API](http://nginx.org/en/docs/http/ngx_http_api_module.html#api). The latter allows the Ingress Controller to dynamically change the upstream servers of an upstream. -- Instead of the stub status metrics, the extended metrics are used, which are available via NGINX Plus API. -- In addition to TLS certs and keys, the IC writes JWKs from the secrets of the type `nginx.org/jwk`, and NGINX workers read them. - -## The Ingress Controller Process - -This section covers the architecture of the IC process, including: - -- How the IC processes a new Ingress resource created by a user. -- The summary of how the IC works and how it relates to Kubernetes Controllers. -- The different components of the IC process. - -### Processing a New Ingress Resource - -The following diagram depicts how the IC processes a new Ingress resource. We represent the NGINX master and worker processes as a single rectangle *NGINX* for simplicity. Also, note that VirtualServer and VirtualServerRoute resources are processed similarly. - -{{< img title="IC process" src="./img/ic-process.png" >}} - -Processing a new Ingress resource involves the following steps, where each step corresponds to the arrow on the diagram with the same number: - -1. *User* creates a new Ingress resource. -2. The IC process has a *Cache* of the resources in the cluster. The *Cache* includes only the resources the IC is interested in, such as Ingresses. The *Cache* stays in sync with the Kubernetes API by [watching for changes to the resources](https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes). -3. Once the *Cache* has the new Ingress resource, it notifies the *Control loop* about the changed resource. -4. The *Control loop* gets the latest version of the Ingress resource from the *Cache*. Because the Ingress resource references other resources, such as TLS Secrets, the *Control loop* gets the latest versions of any referenced resources as well. -5. The *Control loop* generates TLS certificates and keys from the TLS Secrets and writes them to the filesystem. -6. The *Control loop* generates and writes the NGINX *configuration files*, which correspond to the Ingress resource, and writes them to the filesystem. -7. The *Control loop* reloads *NGINX* and waits for *NGINX* to successfully reload. As part of the reload: - 1. *NGINX* reads the *TLS certs and keys*. - 1. *NGINX* reads the *configuration files*. -8. The *Control loop* emits an event for the Ingress resource and updates its status. If the reload fails, the event includes the error message. - -### The Ingress Controller is a Kubernetes Controller - -Based on the example from the previous section, we can generalize how the IC works: - -*The IC constantly processes both new resources and changes to the existing resources in the cluster. As a result, the NGINX configuration stays up-to-date with the resources in the cluster.* - -The IC is an example of a [Kubernetes controller](https://kubernetes.io/docs/concepts/architecture/controller/): the IC runs a control loop that ensures NGINX is configured according to the desired state (Ingresses and other resources). - -The desired state is concentrated in the following built-in Kubernetes resources and Custom Resources (CRs): - -- Layer 7 Load balancing configuration: - - Ingresses - - VirtualServers (CR) - - VirtualServerRoutes (CR) -- Layer 7 policies: - - Policies (CR) -- Layer 4 load balancing configuration: - - TransportServers (CR) -- Service discovery: - - Services - - Endpoints - - Pods -- Secret configuration: - - Secrets -- Global Configuration: - - ConfigMap (only one resource) - - GlobalConfiguration (CR, only one resource) - -The IC can watch additional Custom Resources, which are less common and not enabled by default: - -- [NGINX App Protect resources](/nginx-ingress-controller/app-protect/configuration/) (APPolicies, APLogConfs, APUserSigs) -- IngressLink resource (only one resource) - -In the next section, we examine the different components of the IC process. - -## Ingress Controller Process Components - -In this section, we describe the components of the IC process and how they interact, including: - -1. How the IC watches for resources changes. -1. The main components of the IC control loop. -1. How those components process a resource change. -1. A few additional components, which are crucial for processing changes. - -The IC is written in [go](https://golang.org/) and relies heavily on the [Go client for Kubernetes](https://github.com/kubernetes/client-go). In the sections next, we include links to the code on GitHub when necessary. - -### Resource Caches - -In the section [Processing a New Ingress Resource](#processing-a-new-ingress-resource), we mentioned that the IC has a cache of the resources in the cluster that stays in sync with the Kubernetes API by watching for changes to the resources. We also mentioned that once cache is updated, it notifies the control loop about the changed resource. - -The cache is actually a collection of *informers*. The following diagram shows how changes to resources are processed by the IC. - -{{< img title="IC process components" src="./img/ic-process-components.png" >}} - -- For every resource type the IC monitors, it creates an [*Informer*](https://pkg.go.dev/k8s.io/client-go@v0.21.0/tools/cache#SharedInformer). The *Informer* includes a *Store* that holds the resources of that type. To keep the *Store* in sync with the latest versions of the resources in the cluster, the *Informer* calls the Watch and List *Kubernetes APIs* for that resource type (see the arrow *1. Watch and List* on the diagram). -- When a change happens in the cluster (for example, a new resource is created), the *Informer* updates its *Store* and invokes [*Handlers*](https://pkg.go.dev/k8s.io/client-go@v0.21.0/tools/cache#ResourceEventHandler) (see the arrow *2. Invoke*) for that *Informer*. -- The IC registers handlers for every *Informer*. Most of the time, a *Handler* creates an entry for the affected resource in the *Workqueue* where a workqueue element includes the type of the resource and its namespace and name. (See the arrow *3. Put*.) -- The *Workqueue* always tries to drain itself: if there is an element at the front, the queue will remove the element and send it to the *Controller* by calling a callback function. (See the arrow *4. Send*.) -- The *Controller* is the primary component in the IC, which represents the control loop. We explain the components in [The Control Loop](#the-control-loop) section. For now, it suffices to know that to process a workqueue element, the *Controller* component gets the latest version of the resource from the *Store* (see the arrow *5. Get*), reconfigures *NGINX* according to the resource (see the arrow *6. Reconfigure*), updates the resource status, and emits an event via the *Kubernetes API* (see the arrow *7. Update status and emit event*). - -### The Control Loop - -This section discusses the main components of the IC, which comprise the control loop: - -- [Controller](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/controller.go#L90) - - Runs the IC control loop. - - Instantiates *Informers*, *Handlers*, the *Workqueue* and additional helper components. - - Includes the sync method (see the next section), which is called by the *Workqueue* to process a changed resource. - - Passes changed resources to *Configurator* to re-configure NGINX. -- [Configurator](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/configs/configurator.go#L95) - - Generates NGINX configuration files, TLS and cert keys, and JWKs based on the Kubernetes resource. - - Uses *Manager* to write the generated files and reload NGINX. -- [Manager](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/nginx/manager.go#L52) - - Controls the lifecycle of NGINX (starting, reloading, quitting). See [Reloading NGINX](#reloading-nginx) for more details about reloading. - - Manages the configuration files, TLS keys and certs, and JWKs. - -The following diagram shows how the three components interact: - -{{< img src="./img/control-loop.png" title="Control Loop" >}} - -#### The Controller Sync Method - -The Controller [sync](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/controller.go#L663) method is called by the *Workqueue* to process a change of a resource. The method determines the *kind* of the resource and calls the appropriate *sync* method (for example, *syncIngress* for Ingresses). - -Rather than show how all the various sync methods work, we focus on the most important one -- the *syncIngress* method -- and look at how it processes a new Ingress resource, illustrated in the diagram below. - -{{< img src="img/controller-sync.png" title="Controller sync" >}} - -1. The *Workqueue* calls the *sync* method and passes a workqueue element to it that includes the changed resource *kind* and *key* (the key is the resource namespace/name like “default/cafe-ingress”). -2. Using the *kind*, the *sync* method calls the appropriate sync method and passes the resource key. For Ingresses, that method is *syncIngress*. -3. *syncIngress* gets the Ingress resource from the *Ingress Store* using the key. The *Store* is controlled by the *Ingress Informer*, as mentioned in the section [Resource Caches](#resource-caches). **Note**: In the code, we use the helper *storeToIngressLister* type that wraps the *Store*. -4. *syncIngress* calls *AddOrUpdateIngress* of the *Configuration*, passing the Ingress along. The [Configuration](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/configuration.go#L320) is a component that represents a valid collection of load balancing configuration resources (Ingresses, VirtualServers, VirtualServerRoutes, TransportServers), ready to be converted to the NGINX configuration (see the [Configuration section](#configuration) for more details). *AddOrUpdateIngress* returns a list of [ResourceChanges](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/configuration.go#L59), which must be reflected in the NGINX config. Typically, for a new Ingress resource, the *Configuration* returns only a single *ResourceChange*. -5. *syncIngress* calls *processChanges*, which processes the single Ingress *ResourceChange*. - 1. *processChanges* creates an extended Ingress resource (*IngressEx*) that includes the original Ingress resource and its dependencies, such as Endpoints and Secrets, to generate the NGINX configuration. For simplicity, we don’t show this step on the diagram. - 2. *processChanges* calls *AddOrUpdateIngress* of the *Configurator* and passes the extended Ingress resource. -6. *Configurator* generates an NGINX configuration file based on the extended Ingress resource and then: - 1. Calls *Manager’s CreateConfig()* to update the config for the Ingress resource. - 2. Calls *Manager’s Reload()* to reload NGINX. -7. The reload status is propagated from *Manager* to *processChanges*. The status is either a success or a failure with an error message. -8. *processChanges* calls *updateRegularIngressStatusAndEvent* to update the status of the Ingress resource and emit an event with the status of the reload. Both involve making an API call to the Kubernetes API. - -Notes: - -- Some details weren't discussed for simplicity. You can view the source code if you want a fuller picture. -- The *syncVirtualServer*, *syncVirtualServerRoute*, and *syncTransportServer* methods are similar to syncIngress. The other sync methods are different. However, those methods typically involve finding the affected Ingress, VirtualServer, and TransportServer resources and regenerating a configuration for them. -- The *Workqueue* has only a single worker thread that calls the sync method synchronously. This means that the control loop processes only one change at a time. - -#### Helper Components - -There are two additional helper components crucial for processing changes: *Configuration* and *LocalSecretStore*. - -##### Configuration - -[*Configuration*](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/configuration.go#L320) holds the latest valid state of the IC load balancing configuration resources: Ingresses, VirtualServers, VirtualServerRoutes, TransportServers, and GlobalConfiguration. - -The *Configuration* supports add (for add/update) and delete operations on the resources. When you add/update/delete a resource in the Configuration, it performs the following: - -1. Validates the object (for add/update) -2. Calculates the changes to the affected resources that are necessary to propagate to the NGINX config, returning the changes to the caller. - -For example, when you add a new Ingress resource, the *Configuration* returns a change requiring the IC to add the configuration for that Ingress to the NGINX config files. Another example: if you make an existing Ingress resource invalid, the *Configuration* returns a change requiring the IC to remove the configuration for that Ingress from the NGINX config files. - -Additionally, the *Configuration* ensures that only one Ingress/VirtualServer/TransportServer (TLS Passthrough) holds a particular host (for example, cafe.example.com) and only one TransportServer (TCP/UDP) holds a particular listener (for example, port 53 for UDP). This ensures that no host or listener collisions happen in the NGINX config. - -Ultimately, the IC ensures the NGINX config on the filesystem reflects the state of the objects in the *Configuration* at any point in time. - -##### LocalSecretStore - -[*LocalSecretStore*](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/secrets/store.go#L32) (of the *SecretStore* interface) holds the valid Secret resources and keeps the corresponding files on the filesystem in sync with them. Secrets are used to hold TLS certificates and keys (type `kubernetes.io/tls`), CAs (`nginx.org/ca`), JWKs (`nginx.org/jwk`), and client secrets for an OIDC provider (`nginx.org/oidc`). - -When *Controller* processes a change to a configuration resource like Ingress, it creates an extended version of a resource that includes the dependencies -- such as Secrets -- necessary to generate the NGINX configuration. *LocalSecretStore* allows *Controller* to get a reference on the filesystem for a secret by the secret key (namespace/name). - -## Reloading NGINX - -The following section covers reloading NGINX in general and specifically how the Ingress Controller implements it. - -### Reloading in General - -Reloading NGINX is necessary to apply the new configuration and involves the following actions: - -1. The administrator sends a HUP (hangup) signal to the NGINX master process to trigger a reload. -1. The master process brings down the worker processes with the old configuration and starts worker processes with the new configuration. -1. The administrator verifies the reload has successfully finished. - -> Refer to the [NGINX documentation](https://nginx.org/en/docs/control.html#reconfiguration) for more details about reloading. See also [this blog post](https://www.nginx.com/blog/inside-nginx-how-we-designed-for-performance-scale/) for an overview of the NGINX architecture. - -#### How to Reload - -The NGINX binary (`nginx`) supports the reload operation with the `-s reload` option. When you run this option: - -1. It validates the new NGINX configuration and exits if it is invalid printing the error messages to the stderr. -1. It sends a HUP signal to the NGINX master process and exits. - -Alternatively, you can send a HUP signal to the NGINX master process directly. - -#### How to Confirm Reloading Success - -`nginx -s reload` doesn't wait for NGINX to finish reloading. As a result, it is the responsibility of the administrator to confirm it. There are a few options: - -- Check if the master process created new worker processes. For example, by running `ps` or reading the `/proc` file system. -- Send an HTTP request to NGINX, and if a new worker process responds, you'll know the NGINX reloaded successfully. Note: this requires additional NGINX config, see the [Reloading in the Ingress Controller section](#reloading-in-the-ic). - -Reloading takes time, usually at least 200ms. The time depends on the configuration size, the number of TLS certificates/keys, enabled modules, configuration details, and the available CPU resources. - -#### Potential Problems - -Most of the time, if `nginx -s reload` succeeds, the reload will also succeed. In rare cases the reload fails, the NGINX master will print the error message to the error log. For example: - -``` -2022/07/09 00:56:42 [emerg] 1353#1353: limit_req "one" uses the "$remote_addr" key while previously it used the "$binary_remote_addr" key -``` - -The operation is graceful; reloading doesn't lead to any traffic loss by NGINX. However, frequent reloads can lead to high memory utilization and potentially NGINX stopping with an OOM (Out-Of-Memory) error, resulting in traffic loss. This can happen if you (1) proxy traffic that utilizes long-lived connections (ex: Websockets, gRPC) and (2) reload frequently. In this case, you can end up with multiple generations of NGINX worker processes that are shutting down (old NGINX workers will not shut down until all connections are terminated either by clients or backends, unless you configure [worker_shutdown_timeout](https://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout) which will force old workers to shut down after the timeout). Eventually, all those worker processes can exhaust the system's available memory. - -Since both the old and new NGINX worker processes coexist during a reload, reloading can lead to a spike in memory utilization up to two times. Because of the lack of available memory, the NGINX master process can fail to create new worker processes. - -### Reloading in the IC - -The Ingress Controller reloads NGINX to apply configuration changes. - -To facilitate reloading, the Ingress Controller configures a server listening on the Unix socket `unix:/var/lib/nginx/nginx-config-version.sock` that responds with the config version for `/configVersion` URI. The Ingress Controller writes the config to `/etc/nginx/config-version.conf`. - -A reload involves multiple steps: - -1. The Ingress Controller updates generated configuration files, including any secrets. -1. The Ingress Controller updates the config version in `/etc/nginx/config-version.conf`. -1. The Ingress Controller runs `nginx -s reload`. If the command fails, the Ingress Controller logs the error and considers the reload failed. -2. Assuming the command succeeds, the Ingress Controller periodically checks for the config version by sending an HTTP request to the config version server on `unix:/var/lib/nginx/nginx-config-version.sock`. -3. Once the Ingress Controller sees the correct config version returned by NGINX, it considers the reload successful. If it doesn't see the correct config version after the configurable timeout (see `-nginx-reload-timeout` [cli argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments), the Ingress Controller considers the reload failed. - -> The [Ingress Controller Control Loop](#the-control-loop) stops during a reload so that it cannot change any configuration files or reload NGINX until the current reload succeeds or fails. - -### When the Ingress Controller Reloads NGINX - -The Ingress Controller reloads NGINX every time the Control Loop processes a change that affects the generated NGINX configuration. In general, every time a resource is changed, the Ingress Controller will regenerate the configuration and reload NGINX. A resource could be of any type the Ingress Controller monitors -- see [The Ingress Controller is a Kubernetes Controller](#the-ingress-controller-is-a-kubernetes-controller) section. - -There are three special cases: - -- *Start*. When the Ingress Controller starts, it processes all resources in the cluster and only then reloads NGINX. This avoids creating a "reload storm" by reloading only once. -- *Batch updates*. When the Ingress Controller receives a number of concurrent requests from the Kubernetes API, it will pause NGINX reloads until the task queue is empty. This reduces the number of reloads to minimize the impact of batch updates and reduce the risk of OOM (Out of Memory) errors. -- *NGINX Plus*. If the Ingress Controller uses NGINX Plus, it will not reload NGINX Plus for changes to the Endpoints resources. In this case, the Ingress Controller will use the NGINX Plus API to update the corresponding upstreams and skip reloading. diff --git a/docs/content/intro/nginx-plus.md b/docs/content/intro/nginx-plus.md deleted file mode 100644 index d89eac745c7..00000000000 --- a/docs/content/intro/nginx-plus.md +++ /dev/null @@ -1,35 +0,0 @@ ---- -title: NGINX Ingress Controller with NGINX Plus -description: "This document explains the key characteristics that NGINX Plus brings on top of NGINX into the NGINX Ingress Controller." -weight: 400 -doctypes: ["concept"] -toc: true -docs: "DOCS-611" -aliases: - - /nginx-plus/ ---- - - -NGINX Ingress Controller works with both [NGINX](https://nginx.org/) and [NGINX Plus](https://www.nginx.com/products/nginx/) -- a commercial closed source version of NGINX that comes with additional features and support. - -Below are the key characteristics that NGINX Plus brings on top of NGINX into the NGINX Ingress Controller. - -## Additional features - -- *Real-time metrics* A number metrics about how NGINX Plus and applications are performing are available through the API or a [built-in dashboard](https://docs.nginx.com/nginx-ingress-controller/logging-and-monitoring/status-page/). Optionally, the metrics can be exported to [Prometheus](https://docs.nginx.com/nginx-ingress-controller/logging-and-monitoring/prometheus/). -- *Additional load balancing methods*. The following additional methods are available: `least_time` and `random two least_time` and their derivatives. See the [documentation](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) for the complete list of load balancing methods. -- *Session persistence* The *sticky cookie* method is available. See the [Session Persistence for VirtualServer Resources example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/session-persistence) and the [Session Persistence for Ingress Resources example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/session-persistence). -- *Active health checks*. See the [Support for Active Health Checks for VirtualServer Resources example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/health-checks) and the [Support for Active Health Checks for Ingress Resources example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). -- *JWT validation*. See the [Support for JSON Web Tokens for VirtualServer Resources example (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/jwt) and the [Support for JSON Web Tokens for Ingress Resources example (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). - -See the [VirtualServer]({{< relref "configuration/virtualserver-and-virtualserverroute-resources" >}}), [Policy]({{< relref "configuration/policy-resource" >}}) and [TransportServer]({{< relref "configuration/transportserver-resource" >}}) docs for a comprehensive guide of the NGINX Plus features available by using our custom resources - -For the complete list of available NGINX Plus features available for Ingress resources, see the [ConfigMap]({{< relref "configuration/global-configuration/configmap-resource">}}) and [Annotations]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations">}}) docs. Note that such features are configured through annotations that start with `nginx.com`, for example, `nginx.com/health-checks`. - -## Dynamic reconfiguration - -Every time the number of pods of services you expose via an Ingress resource changes, the Ingress Controller updates the configuration of the load balancer to reflect those changes. For NGINX, the configuration file must be changed and the configuration subsequently reloaded. For NGINX Plus, the dynamic reconfiguration is utilized, which allows NGINX Plus to be updated on-the-fly without reloading the configuration. This prevents increase of memory usage during reloads, especially with a high volume of client requests, as well as increased memory usage when load balancing applications with long-lived connections (WebSocket, applications with file uploading/downloading or streaming). - -## Commercial support - -Support from NGINX Inc is available for NGINX Plus Ingress Controller. diff --git a/docs/content/intro/overview.md b/docs/content/intro/overview.md deleted file mode 100644 index 1dd0e18799b..00000000000 --- a/docs/content/intro/overview.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -title: Overview -description: Overview of the NGINX Ingress Controller -weight: 100 -doctypes: [""] -toc: true -docs: "DOCS-612" ---- - - - -The NGINX Ingress Controller an implementation of a Kubernetes Ingress Controller for NGINX and NGINX Plus. - -## What is the Ingress? - -The Ingress is a Kubernetes resource that lets you configure an HTTP load balancer for applications running on Kubernetes, represented by one or more [Services](https://kubernetes.io/docs/concepts/services-networking/service/). Such a load balancer is necessary to deliver those applications to clients outside of the Kubernetes cluster. - -The Ingress resource supports the following features: - -- **Content-based routing**: - - *Host-based routing*. For example, routing requests with the host header `foo.example.com` to one group of services and the host header `bar.example.com` to another group. - - *Path-based routing*. For example, routing requests with the URI that starts with `/serviceA` to service A and requests with the URI that starts with `/serviceB` to service B. -- **TLS/SSL termination** for each hostname, such as `foo.example.com`. - -See the [Ingress Reference Documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) to learn more about the Ingress resource. - -## What is the Ingress Controller? - -The Ingress Controller is an application that runs in a cluster and configures an HTTP load balancer according to Ingress resources. The load balancer can be a software load balancer running in the cluster or a hardware or cloud load balancer running externally. Different load balancers require different Ingress Controller implementations. - -In the case of NGINX, the Ingress Controller is deployed in a pod along with the load balancer. - -## NGINX Ingress Controller - -NGINX Ingress Controller works with both NGINX and NGINX Plus and supports the standard Ingress features - content-based routing and TLS/SSL termination. - -Additionally, several NGINX and NGINX Plus features are available as extensions to the Ingress resource via annotations and the ConfigMap resource. In addition to HTTP, NGINX Ingress Controller supports load balancing Websocket, gRPC, TCP and UDP applications. See [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) and [Annotations](/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations) docs to learn more about the supported features and customization options. - -As an alternative to the Ingress, NGINX Ingress Controller supports the VirtualServer and VirtualServerRoute resources. They enable use cases not supported with the Ingress resource, such as traffic splitting and advanced content-based routing. See [VirtualServer and VirtualServerRoute Resources doc](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources). - -TCP, UDP and TLS Passthrough load balancing is also supported. See the [TransportServer resource doc](/nginx-ingress-controller/configuration/transportserver-resource/). diff --git a/docs/content/logging-and-monitoring/prometheus.md b/docs/content/logging-and-monitoring/prometheus.md index a669b0255b0..2d639639242 100644 --- a/docs/content/logging-and-monitoring/prometheus.md +++ b/docs/content/logging-and-monitoring/prometheus.md @@ -3,9 +3,9 @@ title: Prometheus description: "The Ingress Controller exposes a number of metrics in the Prometheus format." weight: 2000 -doctypes: ["concept"] +doctypes: [""] aliases: - - /prometheus/ + - /prometheus/ toc: true docs: "DOCS-614" --- @@ -15,8 +15,6 @@ The Ingress Controller exposes a number of metrics in the [Prometheus](https://p ## Enabling Metrics -### Using Manifest - If you're using *Kubernetes manifests* (Deployment or DaemonSet) to install the Ingress Controller, to enable Prometheus metrics: 1. Run the Ingress Controller with the `-enable-prometheus-metrics` [command-line argument](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments). As a result, the Ingress Controller will expose NGINX or NGINX Plus metrics in the Prometheus format via the path `/metrics` on port `9113` (customizable via the `-prometheus-metrics-listen-port` command-line argument). @@ -37,24 +35,8 @@ If you're using *Kubernetes manifests* (Deployment or DaemonSet) to install the prometheus.io/scheme: http ``` -### Using Helm - If you're using *Helm* to install the Ingress Controller, to enable Prometheus metrics, configure the `prometheus.*` parameters of the Helm chart. See the [Installation with Helm](/nginx-ingress-controller/installation/installation-with-helm) doc. -### Using ServiceMonitor - -When deploying with *Helm*, you can deploy a `Service` and `ServiceMonitor` resource using the `prometheus.service.*` and `prometheus.serviceMonitor.*` parameters. -When these resources are deployed, Prometheus metrics exposed by the NGINX Ingress Controller can be captured and enumerated using a `Prometheus` resource alongside a Prometheus Operator deployment. - -To view metrics captured this way, the following is required: -* The latest ServiceMonitor CRD from the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) repository: -```shell -LATEST=$(curl -s https://api.github.com/repos/prometheus-operator/prometheus-operator/releases/latest | jq -cr .tag_name) -curl https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$LATEST/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml | kubectl create -f - -``` -* A working [Prometheus resource and Prometheus Operator](https://prometheus-operator.dev/docs/user-guides/getting-started/) - - ## Available Metrics The Ingress Controller exports the following metrics: diff --git a/docs/content/intro/_index.md b/docs/content/overview/_index.md similarity index 72% rename from docs/content/intro/_index.md rename to docs/content/overview/_index.md index 6a9784506c0..05b03b9343b 100644 --- a/docs/content/intro/_index.md +++ b/docs/content/overview/_index.md @@ -1,7 +1,7 @@ --- -title: Intro +title: Overview description: -weight: 1200 +weight: 100 menu: docs: parent: NGINX Ingress Controller diff --git a/docs/content/overview/about.md b/docs/content/overview/about.md new file mode 100644 index 00000000000..8b8ada021f6 --- /dev/null +++ b/docs/content/overview/about.md @@ -0,0 +1,16 @@ +--- +title: About +description: "This document describes the F5 NGINX Ingress Controller, an Ingress Controller implementation for NGINX and NGINX Plus." +weight: 100 +doctypes: ["concept"] +docs: "DOCS-612" +--- + +
+ +The NGINX Ingress Controller is an [Ingress Controller]({{< relref "glossary.md#ingress-controller">}}) implementation for NGINX and NGINX Plus that can load balance Websocket, gRPC, TCP and UDP applications. It supports standard [Ingress]({{< relref "glossary.md#ingress">}}) features such as content-based routing and TLS/SSL termination. Several NGINX and NGINX Plus features are available as extensions to Ingress resources through [Annotations]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations">}}) and the [ConfigMap]({{< relref "configuration/global-configuration/configmap-resource">}}) resource. + +The NGINX Ingress Controller supports the [VirtualServer and VirtualServerRoute resources]({{< relref "configuration/virtualserver-and-virtualserverroute-resources">}}) as alternatives to Ingress, enabling traffic splitting and advanced content-based routing. It also supports TCP, UDP and TLS Passthrough load balancing using [TransportServer resources]({{< relref "configuration/transportserver-resource">}}). + +To learn more about the NGINX Ingress Controller, please read the [How NGINX Ingress Controller is Designed +]({{< relref "overview/design">}}) and [Extensibility with NGINX Plus]({{< relref "overview/nginx-plus">}}) pages. diff --git a/docs/content/intro/control-loop.png b/docs/content/overview/control-loop.png similarity index 100% rename from docs/content/intro/control-loop.png rename to docs/content/overview/control-loop.png diff --git a/docs/content/intro/nginx-ingress-controllers.md b/docs/content/overview/controller-comparison.md similarity index 94% rename from docs/content/intro/nginx-ingress-controllers.md rename to docs/content/overview/controller-comparison.md index 2d23e277828..ff506c17fe2 100644 --- a/docs/content/intro/nginx-ingress-controllers.md +++ b/docs/content/overview/controller-comparison.md @@ -1,16 +1,12 @@ --- title: Which Ingress Controller Do I Need? -description: -weight: 200 -draft: true +description: This document describes the key differences between the community Ingress-NGINX Controller and F5 NGINX Ingress Controller. +weight: 400 doctypes: ["concept"] toc: true docs: "DOCS-610" -aliases: - - /nginx-ingress-controllers/ --- - There are two NGINX-based Ingress Controller implementations out there: the one made by NGINX ([nginxinc/kubernetes-ingress](https://github.com/nginxinc/kubernetes-ingress)) and the one made by Kubernetes ([kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx)). In this document, we explain the key differences between those implementations. This information should help you to choose an appropriate implementation for your requirements or move from one implementation to the other. ## Which One Am I Using? @@ -29,11 +25,11 @@ The table below summarizes the key difference between nginxinc/kubernetes-ingres | NGINX version | [Custom](https://github.com/kubernetes/ingress-nginx/tree/main/images/nginx) NGINX build that includes several third-party modules | NGINX official mainline [build](https://github.com/nginxinc/docker-nginx) | NGINX Plus | | Commercial support | N/A | N/A | Included | | **Load balancing configuration via the Ingress resource** | -| Merging Ingress rules with the same host | Supported | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/mergeable-ingress-types) | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/mergeable-ingress-types) | +| Merging Ingress rules with the same host | Supported | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/mergeable-ingress-types) | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/mergeable-ingress-types) | | HTTP load balancing extensions - Annotations | See the [supported annotations](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/) | See the [supported annotations](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/) | See the [supported annotations](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/)| | HTTP load balancing extensions -- ConfigMap | See the [supported ConfigMap keys](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/) | See the [supported ConfigMap keys](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) | See the [supported ConfigMap keys](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) | | TCP/UDP | Supported via a ConfigMap | Supported via custom resources | Supported via custom resources | -| Websocket | Supported | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/websocket) | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/websocket) | +| Websocket | Supported | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/websocket) | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/websocket) | | TCP SSL Passthrough | Supported via a ConfigMap | Supported via custom resources | Supported via custom resources | | JWT validation | Not supported | Not supported | Supported | | Session persistence | Supported via a third-party module | Not supported | Supported | diff --git a/docs/content/intro/controller-sync.png b/docs/content/overview/controller-sync.png similarity index 100% rename from docs/content/intro/controller-sync.png rename to docs/content/overview/controller-sync.png diff --git a/docs/content/overview/design.md b/docs/content/overview/design.md new file mode 100644 index 00000000000..b471b1cd150 --- /dev/null +++ b/docs/content/overview/design.md @@ -0,0 +1,312 @@ +--- +title: How NGINX Ingress Controller is Designed +description: "This document explains how the F5 NGINX Ingress Controller is designed, and how it works with NGINX and NGINX Plus." +weight: 200 +doctypes: ["reference"] +toc: true +docs: "DOCS-609" +--- + +
+ +The intended audience for this information is primarily the two following groups: + +- _Operators_ who want to know how the software works and understand how it can fail. +- _Developers_ who want to [contribute](https://github.com/nginxinc/kubernetes-ingress/blob/main/CONTRIBUTING.md) to the project. + +We assume that the reader is familiar with core Kubernetes concepts, such as Pods, Deployments, Services, and Endpoints. For an understanding of how NGINX itself works, you can read the ["Inside NGINX: How We Designed for Performance & Scale"](https://www.nginx.com/blog/inside-nginx-how-we-designed-for-performance-scale/) blog post. + +For conciseness in diagrams, NGINX Ingress Controller is often labelled "IC" on this page. + +## NGINX Ingress Controller at a High Level + +This figure depicts an example of NGINX Ingress Controller exposing two web applications within a Kubernetes cluster to clients on the internet: + +{{}} + +{{}} For simplicity, necessary Kubernetes resources like Deployments and Services aren't shown, which Admin and the users also need to create.{{}} + +The figure shows: + +- A _Kubernetes cluster_. +- Cluster users _Admin_, _User A_ and _User B_, which use the cluster via the _Kubernetes API_. +- _Clients A_ and _Clients B_, which connect to the _Applications A_ and _B_ deployed by the corresponding users. +- _NGINX Ingress Controller_, deployed in a pod with the namespace _nginx-ingress_ and configured using the _ConfigMap resource_ _nginx-ingress_. A single pod is depicted; at least two pods are typically deployed for redundancy. _NGINX Ingress Controller_ uses the _Kubernetes API_ to get the latest Ingress resources created in the cluster and then configures _NGINX_ according to those resources. +- _Application A_ with two pods deployed in the _namespace A_ by _User A_. To expose the application to its clients (_Clients A_) via the host `a.example.com`, _User A_ creates _Ingress A_. +- _Application B_ with one pod deployed in the _namespace B_ by _User B_. To expose the application to its clients (_Clients B_) via the host `b.example.com`, _User B_ creates _VirtualServer B_. +- _Public Endpoint_, which fronts the _NGINX Ingress Controller_ pod(s). This is typically a standalone TCP load balancer (Cloud, software, or hardware) or a combination of a load balancer with a NodePort service. _Clients A_ and _B_ connect to their applications via the _Public Endpoint_. + +The yellow and purple arrows represent connections related to the client traffic, and the black arrows represent access to the Kubernetes API. + +## The NGINX Ingress Controller Pod + +The NGINX Ingress Controller pod consists of a single container, which includes the following: + +- The _NGINX Ingress Controller process_, which configures NGINX according to Ingress and other resources created in the cluster. +- The _NGINX master process_, which controls NGINX worker processes. +- _NGINX worker processes_, which handle the client traffic and load balance the traffic to the backend applications. + +The following is an architectural diagram depicting how those processes interact together and with some external entities: + +{{}} + +This table describes each connection, starting with its type: + + +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} +| # | Protocols | Description | +| --- | --- | --- | +|1|HTTP| _Prometheus_ fetches NGINX Ingress Controller and NGINX metrics with an NGINX Ingress Controller HTTP endpoint (Default `:9113/metrics`). **Note**: *Prometheus* is not required and the endpoint can be turned off. | +|2|HTTPS| _NGINX Ingress Controller_ reads the _Kubernetes API_ for the latest versions of the resources in the cluster and writes to the API to update the handled resources' statuses and emit events. +|3|HTTP| _Kubelet_ checks the _NGINX Ingress Controller_ readiness probe (Default `:8081/nginx-ready`) to consider the _NGINX Ingress Controller_ pod [ready](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-conditions). +|4|File I/O| When _NGINX Ingress Controller_ starts, it reads the _configuration templates_ from the filesystem necessary for configuration generation. The templates are located in the `/` directory of the container and have the `.tmpl` extension +|5|File I/O| _NGINX Ingress Controller_ writes logs to *stdout* and *stderr*, which are collected by the container runtime. +|6|File I/O| _NGINX Ingress Controller_ generates NGINX *configuration* based on the resources created in the cluster (See [NGINX Ingress Controller is a Kubernetes Controller](#nginx-ingress-controller-is-a-kubernetes-controller)) and writes it on the filesystem in the `/etc/nginx` folder. The configuration files have a `.conf` extension. +|7|File I/O| _NGINX Ingress Controller_ writes _TLS certificates_ and _keys_ from any [TLS Secrets](https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets) referenced in the Ingress and other resources to the filesystem. +|8|HTTP| _NGINX Ingress Controller_ fetches the [NGINX metrics](https://nginx.org/en/docs/http/ngx_http_stub_status_module.html#stub_status) via the `unix:/var/lib/nginx/nginx-status.sock` UNIX socket and converts it to Prometheus format used in #1. +|9|HTTP| To verify a successful configuration reload, _NGINX Ingress Controller_ ensures at least one _NGINX worker_ has the new configuration. To do that, the *IC* checks a particular endpoint via the `unix:/var/lib/nginx/nginx-config-version.sock` UNIX socket. +|10|N/A| To start NGINX, NGINX Ingress Controller runs the `nginx` command, which launches the _NGINX master_. +|11|Signal| To reload NGINX, the _NGINX Ingress Controller_ runs the `nginx -s reload` command, which validates the configuration and sends the [reload signal](https://nginx.org/en/docs/control.html) to the *NGINX master*. +|12|Signal| To shutdown NGINX, the _NGINX Ingress Controller_ executes `nginx -s quit` command, which sends the graceful shutdown signal to the *NGINX master*. +|13|File I/O| The _NGINX master_ sends logs to its _stdout_ and _stderr_, which are collected by the container runtime. +|14|File I/O| The _NGINX master_ reads the _TLS cert and keys_ referenced in the configuration when it starts or reloads. +|15|File I/O| The _NGINX master_ reads _configuration files_ when it starts or during a reload. +|16|Signal| The _NGINX master_ controls the [lifecycle of _NGINX workers_](https://nginx.org/en/docs/control.html#reconfiguration) it creates workers with the new configuration and shutdowns workers with the old configuration. +|17|File I/O| An _NGINX worker_ writes logs to its _stdout_ and _stderr_, which are collected by the container runtime. +|18|UDP| An _NGINX worker_ sends the HTTP upstream server response latency logs via the Syslog protocol over the UNIX socket `/var/lib/nginx/nginx-syslog.sock` to _NGINX Ingress Controller_. In turn, _NGINX Ingress Controller_ analyzes and transforms the logs into Prometheus metrics. +|19|HTTP,HTTPS,TCP,UDP| A _client_ sends traffic to and receives traffic from any of the _NGINX workers_ on ports 80 and 443 and any additional ports exposed by the [GlobalConfiguration resource](/nginx-ingress-controller/configuration/global-configuration/globalconfiguration-resource). +|20|HTTP,HTTPS,TCP,UDP| An _NGINX worker_ sends traffic to and receives traffic from the _backends_. +|21|HTTP| _Admin_ can connect to the [NGINX stub_status](http://nginx.org/en/docs/http/ngx_http_stub_status_module.html#stub_status) using port 8080 via an _NGINX worker_. By default, NGINX only allows connections from `localhost`. +{{% /bootstrap-table %}} + +### Differences with NGINX Plus + +The previous diagram depicts NGINX Ingress Controller using NGINX. NGINX Ingress Controller with NGINX Plus has the following differences: + +- To configure NGINX Plus, NGINX Ingress Controller uses [configuration reloads](#reloading-nginx) and the [NGINX Plus API](http://nginx.org/en/docs/http/ngx_http_api_module.html#api). This allows NGINX Ingress Controller to dynamically change the upstream servers. +- Instead of the stub status metrics, the extended metrics available from the NGINX Plus API are used. +- In addition to TLS certs and keys, NGINX Ingress Controllerf writes JWKs from the secrets of the type `nginx.org/jwk`, and NGINX workers read them. + +## The NGINX Ingress Controller Process + +This section covers the architecture of the NGINX Ingress Controller process, including: + +- How NGINX Ingress Controller processes a new Ingress resource created by a user. +- A summary of how NGINX Ingress Controller works in relation to others Kubernetes Controllers. +- The different components of the IC process. + +### Processing a New Ingress Resource + +The following diagram depicts how NGINX Ingress Controller processes a new Ingress resource. The the NGINX master and worker processes are represented as a single rectangle, _NGINX_ for simplicity. VirtualServer and VirtualServerRoute resources are indicated similarly. + +{{}} + +Processing a new Ingress resource involves the following steps: each step corresponds to the arrow on the diagram with the same number: + +1. _User_ creates a new Ingress resource. +1. The NGINX Ingress Controller process has a _Cache_ of the resources in the cluster. The _Cache_ includes only the resources NGINX Ingress Controller is concerned with such as Ingresses. The _Cache_ stays in sync with the Kubernetes API by [watching for changes to the resources](https://kubernetes.io/docs/reference/using-api/api-concepts/#efficient-detection-of-changes). +1. Once the _Cache_ has the new Ingress resource, it notifies the _Control Loop_ about the changed resource. +1. The _Control Loop_ gets the latest version of the Ingress resource from the _Cache_. Since the Ingress resource references other resources, such as TLS Secrets, the _Control loop_ gets the latest versions of those referenced resources as well. +1. The _Control Loop_ generates TLS certificates and keys from the TLS Secrets and writes them to the filesystem. +1. The _Control Loop_ generates and writes the NGINX _configuration files_, which correspond to the Ingress resource, and writes them to the filesystem. +1. The _Control Loop_ reloads _NGINX_ and waits for _NGINX_ to successfully reload. As part of the reload: + 1. _NGINX_ reads the _TLS certs and keys_. + 1. _NGINX_ reads the _configuration files_. +1. The _Control Loop_ emits an event for the Ingress resource and updates its status. If the reload fails, the event includes the error message. + +### NGINX Ingress Controller is a Kubernetes Controller + +With the context from the previous sections, we can generalize how NGINX Ingress Controller works: + +*NGINX Ingress Controller constantly processes both new resources and changes to the existing resources in the cluster. As a result, the NGINX configuration stays up-to-date with the resources in the cluster.* + +NGINX Ingress Controller is an example of a [Kubernetes Controller](https://kubernetes.io/docs/concepts/architecture/controller/): NGINX Ingress Controller runs a control loop that ensures NGINX is configured according to the desired state (Ingresses and other resources). + +The desired state is based on the following built-in Kubernetes resources and Custom Resources (CRs): + +- Layer 7 Load balancing configuration: + - Ingresses + - VirtualServers (CR) + - VirtualServerRoutes (CR) +- Layer 7 policies: + - Policies (CR) +- Layer 4 load balancing configuration: + - TransportServers (CR) +- Service discovery: + - Services + - Endpoints + - Pods +- Secret configuration: + - Secrets +- Global Configuration: + - ConfigMap (only one resource) + - GlobalConfiguration (CR, only one resource) + +NGINX Ingress Controller can watch additional Custom Resources, which are less common and not enabled by default: + +- [NGINX App Protect resources]({{< relref "installation/integrations/app-protect-dos/configuration" >}}) (APPolicies, APLogConfs, APUserSigs) +- IngressLink resource (only one resource) + +## NGINX Ingress Controller Process Components + +In this section, we describe the components of the NGINX Ingress Controller process and how they interact, including: + +1. How NGINX Ingress Controller watches for resources changes. +1. The main components of the NGINX Ingress Controller _Control Loop_. +1. How those components process a resource change. +1. Additional components that are crucial for processing changes. + +NGINX Ingress Controller is written in [Go](https://golang.org/) and relies heavily on the [Go client for Kubernetes](https://github.com/kubernetes/client-go). Where relevant, we include links to the source code on GitHub. + +### Resource Caches + +In an earlier section, [Processing a New Ingress Resource](#processing-a-new-ingress-resource), we mentioned that NGINX Ingress Controller has a cache of the resources in the cluster that stays in sync with the Kubernetes API by watching them for changes. + +We also mentioned that once the cache is updated, it notifies the control loop about the changed resources. The cache is actually a collection of *informers*. The following diagram shows how changes to resources are processed by NGINX Ingress Controller. + +{{}} + +- For every resource type that NGINX Ingress Controller monitors, it creates an [_Informer_](https://pkg.go.dev/k8s.io/client-go@v0.21.0/tools/cache#SharedInformer). The _Informer_ includes a _Store_ that holds the resources of that type. To keep the _Store_ in sync with the latest versions of the resources in the cluster, the _Informer_ calls the Watch and List _Kubernetes APIs_ for that resource type (see the arrow _1. Watch and List_ on the diagram). +- When a change happens in the cluster (for example, a new resource is created), the _Informer_ updates its _Store_ and invokes [_Handlers_](https://pkg.go.dev/k8s.io/client-go@v0.21.0/tools/cache#ResourceEventHandler) (See the arrow _2. Invoke_) for that _Informer_. +- NGINX Ingress Controller registers _Handlers_ for every _Informer_. Most of the time, a _Handler_ creates an entry for the affected resource in the _Workqueue_ where a workqueue element includes the type of the resource and its namespace and name (See the arrow _3. Put_). +- The _Workqueue_ always tries to drain itself: if there is an element at the front, the queue will remove the element and send it to the _Controller_ by calling a callback function (See the arrow _4. Send_). +- The _Controller_ is the primary component of NGINX Ingress Controller, which represents the _Control Loop_, explained in [The Control Loop](#the-control-loop) section. To process a workqueue element, the _Controller_ component gets the latest version of the resource from the _Store_ (See the arrow _5. Get_), reconfigures _NGINX_ according to the resource (See the arrow _6. Reconfigure*_, updates the resource status, and emits an event via the _Kubernetes API_ (See the arrow _7. Update status and emit event_). + +### The Control Loop + +This section discusses the main components of NGINX Ingress Controller, which comprise the control loop: + +- [Controller](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/controller.go#L90) + - Runs the NGINX Ingress Controller control loop. + - Instantiates _Informers_, _Handlers_, the _Workqueue_ and additional helper components. + - Includes the sync method), which is called by the _Workqueue_ to process a changed resource. + - Passes changed resources to _Configurator_ to re-configure NGINX. +- [Configurator](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/configs/configurator.go#L95) + - Generates NGINX configuration files, TLS and cert keys, and JWKs based on the Kubernetes resource. + - Uses _Manager_ to write the generated files and reload NGINX. +- [Manager](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/nginx/manager.go#L52) + - Controls the lifecycle of NGINX (starting, reloading, quitting). See [Reloading NGINX](#reloading-nginx) for more details about reloading. + - Manages the configuration files, TLS keys and certs, and JWKs. + +The following diagram shows how the three components interact: + +{{}} + +#### The Controller Sync Method + +The Controller [sync](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/controller.go#L663) method is called by the _Workqueue_ to process a change of a resource. The method determines the _kind_ of the resource and calls the appropriate _sync_ method (Such as _syncIngress_ for Ingress resources). + +To explain how the sync methods work, we will examine the most important one: the _syncIngress_ method, and describe how it processes a new Ingress resource. + +{{}} + +1. The _Workqueue_ calls the _sync_ method and passes a workqueue element to it that includes the changed resource _kind_ and _key_ (The key is the resource namespace/name such as “default/cafe-ingress”). +1. Using the _kind_, the _sync_ method calls the appropriate sync method and passes the resource key. For Ingress resources, the method is _syncIngress_. +1. _syncIngress_ gets the Ingress resource from the *Ingress Store* using the key. The _Store_ is controlled by the _Ingress Informer_. In the code, we use the helper _storeToIngressLister_ type that wraps the _Store_. +1. _syncIngress_ calls _AddOrUpdateIngress_ of the _Configuration_, passing the Ingress along. The [Configuration](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/configuration.go#L320) is a component that represents a valid collection of load balancing configuration resources (Ingresses, VirtualServers, VirtualServerRoutes, TransportServers), ready to be converted to the NGINX configuration (see the [Configuration section](#configuration) for more details). _AddOrUpdateIngress_ returns a list of [ResourceChanges](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/configuration.go#L59), which must be reflected in the NGINX config. Typically, for a new Ingress resource, the _Configuration_ returns only a single _ResourceChange_. +1. _syncIngress_ calls _processChanges_, which processes the single Ingress _ResourceChange_. + 1. _processChanges_ creates an extended Ingress resource (_IngressEx_) that includes the original Ingress resource and its dependencies, such as Endpoints and Secrets, to generate the NGINX configuration. For simplicity, we don’t show this step on the diagram. + 1. _processChanges_ calls _AddOrUpdateIngress_ of the _Configurator_ and passes the extended Ingress resource. +1. _Configurator_ generates an NGINX configuration file based on the extended Ingress resource, then: + 1. Calls _Manager’s CreateConfig()_ to update the config for the Ingress resource. + 1. Calls _Manager’s Reload()_ to reload NGINX. +1. The reload status is propagated from _Manager_ to _processChanges_, and is either a success or a failure with an error message. +1. _processChanges_ calls _updateRegularIngressStatusAndEvent_ to update the status of the Ingress resource and emit an event with the status of the reload: both make an API call to the Kubernetes API. + +**Additional Notes**: + +- Many details are not included for conciseness: the source code provides the most granular detail. +- The _syncVirtualServer_, _syncVirtualServerRoute_, and _syncTransportServer_ methods are similar to _syncIngress_, while other sync methods are different. However, those methods typically find the affected Ingress, VirtualServer, and TransportServer resources and regenerate the configuration for them. +- The _Workqueue_ has only a single worker thread that calls the sync method synchronously, meaning the _Control Loop_ processes only one change at a time. + +#### Helper Components + +There are two additional helper components crucial for processing changes: _Configuration_ and _LocalSecretStore_. + +##### Configuration + +[_Configuration_](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/configuration.go#L320) holds the latest valid state of the NGINX Ingress Controller load balancing configuration resources: Ingresses, VirtualServers, VirtualServerRoutes, TransportServers, and GlobalConfiguration. + +The _Configuration_ supports add, update and delete operations on the resources. When you invoke these operations on a resource in the Configuration, it performs the following: + +1. Validates the object (For add or update) +1. Calculates the changes to the affected resources that are necessary to propagate to the NGINX configuration, returning the changes to the caller. + +For example, when you add a new Ingress resource, the _Configuration_ returns a change requiring NGINX Ingress Controller to add the configuration for that Ingress to the NGINX configuration files. If you made an existing Ingress resource invalid, the _Configuration_ returns a change requiring NGINX Ingress Controller to remove the configuration for that Ingress from the NGINX configuration files. + +Additionally, the _Configuration_ ensures that only one Ingress/VirtualServer/TransportServer (TLS Passthrough) resource holds a particular host (For example, cafe.example.com) and only one TransportServer (TCP/UDP) holds a particular listener (Such as port 53 for UDP). This ensures that no host or listener collisions happen in the NGINX configuration. + +Ultimately, NGINX Ingress Controller ensures the NGINX config on the filesystem reflects the state of the objects in the _Configuration_ at any point in time. + +##### LocalSecretStore + +[_LocalSecretStore_](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11.0/internal/k8s/secrets/store.go#L32) (of the _SecretStore_ interface) holds the valid Secret resources and keeps the corresponding files on the filesystem in sync with them. Secrets are used to hold TLS certificates and keys (type `kubernetes.io/tls`), CAs (`nginx.org/ca`), JWKs (`nginx.org/jwk`), and client secrets for an OIDC provider (`nginx.org/oidc`). + +When _Controller_ processes a change to a configuration resource like Ingress, it creates an extended version of a resource that includes the dependencies (Such as Secrets) necessary to generate the NGINX configuration. _LocalSecretStore_ allows _Controller_ to reference the filesystem for a secret using the secret key (namespace/name). + +## Reloading NGINX + +The following sections describe how NGINX reloads and how NGINX Ingress Controller specifically affects this process. + +### How NGINX reloads work + +Reloading NGINX is necessary to apply new configuration changes and occurs with these steps: + +1. The administrator sends a HUP (hangup) signal to the NGINX master process to trigger a reload. +1. The master process brings down the worker processes with the old configuration and starts worker processes with the new configuration. +1. The administrator verifies the reload has successfully finished. + +The [NGINX documentation](https://nginx.org/en/docs/control.html#reconfiguration) has more details about reloading. +#### How to reload NGINX and confirm success + +The NGINX binary (`nginx`) supports the reload operation with the `-s reload` option. When you run this option: + +1. It validates the new NGINX configuration and exits if it is invalid printing the error messages to the stderr. +1. It sends a HUP signal to the NGINX master process and exits. + +As an alternative, you can send a HUP signal to the NGINX master process directly. + +Once the reload operation has been invoked with `nginx -s reload`, there is no wait period for NGINX to finish reloading. This means it is the responsibility of an administator to check it is finished, for which there are a few options: + +- Check if the master process created new worker processes. Two ways are by running `ps` or reading the `/proc` file system. +- Send an HTTP request to NGINX, to see if a new worker process responds. This signifies that NGINX reloaded successfully: this method requires additional NGINX configuration, explained below. + +NGINX reloads take roughly 200ms. The factors affecting reload time are configuration size and details, the number of TLS certificates/keys, enabled modules, and available CPU resources. + +#### Potential problems + +Most of the time, if `nginx -s reload` executes, the reload will also succeed. In the rare case a reload fails, the NGINX master process will print the an error message. This is an example: + +``` +2022/07/09 00:56:42 [emerg] 1353#1353: limit_req "one" uses the "$remote_addr" key while previously it used the "$binary_remote_addr" key +``` + +The operation is graceful; reloading doesn't lead to any traffic loss by NGINX. However, frequent reloads can lead to high memory utilization and potential OOM (Out-Of-Memory) errors, resulting in traffic loss. This can most likely happen if you (1) proxy traffic that utilizes long-lived connections (ex: Websockets, gRPC) and (2) reload frequently. In these scenarios, you can end up with multiple generations of NGINX worker processes that are shutting down which will force old workers to shut down after the timeout). Eventually, all those worker processes can exhaust the system's available memory. + +Old NGINX workers will not shut down until all connections are terminated either by clients or backends, unless you configure [worker_shutdown_timeout](https://nginx.org/en/docs/ngx_core_module.html#worker_shutdown_timeout). Since both the old and new NGINX worker processes coexist during a reload, reloading can lead to two spikes in memory utilization. With a lack of available memory, the NGINX master process can fail to create new worker processes. + +### Reloading in NGINX Ingress Controller + +NGINX Ingress Controller reloads NGINX to apply configuration changes. + +To facilitate reloading, NGINX Ingress Controller configures a server listening on the Unix socket `unix:/var/lib/nginx/nginx-config-version.sock` that responds with the configuration version for `/configVersion` URI. NGINX Ingress Controller writes the configuration to `/etc/nginx/config-version.conf`. + +Reloads occur with this sequence of steps: + +1. NGINX Ingress Controller updates generated configuration files, including any secrets. +1. NGINX Ingress Controller updates the config version in `/etc/nginx/config-version.conf`. +1. NGINX Ingress Controller runs `nginx -s reload`. If the command fails, NGINX Ingress Controller logs the error and considers the reload failed. +1. If the command succeeds, NGINX Ingress Controller periodically checks for the config version by sending an HTTP request to the config version server on `unix:/var/lib/nginx/nginx-config-version.sock`. +1. Once NGINX Ingress Controller sees the correct config version returned by NGINX, it considers the reload successful. If it doesn't see the correct configuration version after the configurable timeout ( [`-nginx-reload-timeout`]({{}})), NGINX Ingress Controller considers the reload failed. + +The [NGINX Ingress Controller Control Loop](#the-control-loop) stops during a reload so that it cannot affect configuration files or reload NGINX until the current reload succeeds or fails. + +### When NGINX Ingress Controller Reloads NGINX + +NGINX Ingress Controller reloads NGINX every time the Control Loop processes a change that affects the generated NGINX configuration. In general, every time a monitored resource is changed, NGINX Ingress Controller will regenerate the configuration and reload NGINX. + +There are three special cases: + +- *Start*. When NGINX Ingress Controller starts, it processes all resources in the cluster and only then reloads NGINX. This avoids a "reload storm" by reloading only once. +- *Batch updates*. When NGINX Ingress Controller receives a number of concurrent requests from the Kubernetes API, it will pause NGINX reloads until the task queue is empty. This reduces the number of reloads to minimize the impact of batch updates and reduce the risk of OOM (Out of Memory) errors. +- *NGINX Plus*. If NGINX Ingress Controller is using NGINX Plus, it will not reload NGINX Plus for changes to the Endpoints resources. In this case, NGINX Ingress Controller will use the NGINX Plus API to update the corresponding upstreams and skip reloading. diff --git a/docs/content/intro/ic-high-level.png b/docs/content/overview/ic-high-level.png similarity index 100% rename from docs/content/intro/ic-high-level.png rename to docs/content/overview/ic-high-level.png diff --git a/docs/content/intro/ic-pod.png b/docs/content/overview/ic-pod.png similarity index 100% rename from docs/content/intro/ic-pod.png rename to docs/content/overview/ic-pod.png diff --git a/docs/content/intro/ic-process-components.png b/docs/content/overview/ic-process-components.png similarity index 100% rename from docs/content/intro/ic-process-components.png rename to docs/content/overview/ic-process-components.png diff --git a/docs/content/intro/ic-process.png b/docs/content/overview/ic-process.png similarity index 100% rename from docs/content/intro/ic-process.png rename to docs/content/overview/ic-process.png diff --git a/docs/content/overview/nginx-plus.md b/docs/content/overview/nginx-plus.md new file mode 100644 index 00000000000..47b06fb9015 --- /dev/null +++ b/docs/content/overview/nginx-plus.md @@ -0,0 +1,33 @@ +--- +title: Extensibility with NGINX Plus +description: "This document explains how F5 NGINX Plus can extend the functionality of the F5 NGINX Ingress Controller." +weight: 300 +doctypes: ["concept"] +docs: "DOCS-611" +aliases: + - /nginx-plus/ +--- + +
+ +The NGINX Ingress Controller works with [NGINX](https://nginx.org/) as well as [NGINX Plus](https://www.nginx.com/products/nginx/), a commercial closed source version of NGINX which has additional features and support from NGINX Inc. The NGINX Ingress Controller can leverage functionality from NGINX Plus to extend its base capabilities. + +## Additional features + +- _Real-time metrics_: Metrics for NGINX Plus and application performance are available through the API or the [NGINX Status Page]({{< relref "logging-and-monitoring/status-page">}}). These metrics can also be exported to [Prometheus]({{< relref "logging-and-monitoring/prometheus">}}). +- _Additional load balancing methods_: The `least_time` and `random two least_time` methods and their derivatives become available. The NGINX [`ngx_http_upstream_module` documentation](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) has the complete list of load balancing methods. +- _Session persistence_: The *sticky cookie* method becomes available. See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/session-persistence) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/session-persistence) examples. +- _Active health checks_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/health-checks) examples. +- _JWT validation_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/jwt) examples. + +For a comprehensive guide of NGINX Plus features available with Ingress resources, see the [ConfigMap]({{< relref "configuration/global-configuration/configmap-resource">}}) and [Annotations]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations">}}) documentation. + +{{< note >}}NGINX Plus features are configured for Ingress resources using Annotations that start with `nginx.com`.{{< /note >}} + +For a comprehensive guide of NGINX Plus features available with custom resources, see the [Policy]({{< relref "configuration/policy-resource" >}}), [VirtualServer]({{< relref "configuration/virtualserver-and-virtualserverroute-resources" >}}) and [TransportServer]({{< relref "configuration/transportserver-resource" >}}) documentation. + +## Dynamic reconfiguration + +The NGINX Ingress Controller updates the configuration of the load balancer to reflect changes every time the number of pods exposed through an Ingress resource changes. When using NGINX, the configuration file must be changed then reloaded. + +For NGINX Plus, its dynamic reconfiguration is utilized, updating NGINX Plus without reloading. This avoids the increase of memory usage caused by reloads (Particularly with large volumes of client requests) and when load balancing applications with long-lived connections (Such as those using WebSockets or handling file uploads, downloads or streaming). diff --git a/docs/content/releases.md b/docs/content/releases.md index d10042ad634..b02292375fd 100644 --- a/docs/content/releases.md +++ b/docs/content/releases.md @@ -1,153 +1,37 @@ --- title: Releases description: "NGINX Ingress Controller Release Notes." -weight: 2100 +weight: 300 doctypes: ["concept"] toc: true docs: "DOCS-616" --- -
-## 3.3.2 - -1 Nov 2023 - -### Fixes - -- [4578](https://github.com/nginxinc/kubernetes-ingress/pull/4578) Update Dockerfile to add user creation for NGINX Plus images. - -### Dependencies - -- [4572](https://github.com/nginxinc/kubernetes-ingress/pull/4572) Update NGINX version to 1.25.3. -- [4569](https://github.com/nginxinc/kubernetes-ingress/pull/4569), [4591](https://github.com/nginxinc/kubernetes-ingress/pull/4591) Bump Go dependencies. - -### Upgrade - -- For NGINX, use the 3.3.2 images from our -[DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.3.2), -[GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), -[Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). -- For NGINX Plus, use the 3.3.2 images from the F5 Container registry, -the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), -the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) -or build your own image using the 3.3.2 source code -- For Helm, use version 1.0.2 of the chart. - -
- -## 3.3.1 - -13 Oct 2023 - -### Overview - -This releases updates NGINX Plus to R30 P1 and dependencies to mitigate HTTP/2 Rapid Reset Attack vulnerability [CVE-2023-44487](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-44487). - -### Dependencies - -- [4501](https://github.com/nginxinc/kubernetes-ingress/pull/4501) Bump Go to 1.21.3 -- [4502](https://github.com/nginxinc/kubernetes-ingress/pull/4502), [4514](https://github.com/nginxinc/kubernetes-ingress/pull/4514) Bump Go dependencies. - -### Upgrade - -- For NGINX, use the 3.3.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.3.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). -- For NGINX Plus, use the 3.3.1 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.3.1 source code - -
- -## 3.3.0 - -26 Sep 2023 - -### Overview - -With release 3.3 the NGINX Ingress Controller continues to advance capabilities for an ever-demanding set of use cases -that go beyond simple layer 7 routing for services running exclusively in Kubernetes. - -When involved in diagnostic operations and viewing the NGINX Plus console or when viewing the enhanced NGINX Plus -metrics through Prometheus, customers now enjoy the added dimension of the backend service being available to aide in -identification of issues as well as observing performance. - -50% of our users continue to rely heavily on the Ingress resource and its "mergeable Ingress" usage pattern, to enhance -the experience for these customers we have added the path-regex annotation with support for case sensitive, case -insensitive, as well as exact regex match patterns. - -Prometheus continues to be the most popular metrics platform for Kubernetes users. To further enhance ease of setting up -integration with Prometheus we have finalized support for the Prometheus serviceMonitor capability. Providing better -scraping controls for Prometheus admins. - -For our most demanding customers performing a blue / green upgrade of the Ingress Controller itself supports the ability -to provide their business customers an enhanced experience with no loss of session fidelity. Support for this pattern -and others has been added through Helm chart enhancement that allows two deployments to share a single ingressClass -resource and duplicate the same configuration. - -To accommodate these enhancements, several new values have been added to our Helm chart, as well as modifications to -existing values. Due to the potential impacts of these changes we have issued a major release to the Helm chart, -advancing to v1.0.0 - -To better align with the demands of supporting additional protocols such as MQTT and QUIC, the NGINX Ingress Controller -is changing how listeners are defined for HTTP traffic. You have always had controls over the ports defined for -TCP/UDP traffic through the GlobalConfiguration and TransportServer objects. That same flexibility has been introduced -for HTTP/S traffic and the VirtualServer. This area will continue to expand to give customers full control over NGINX -listeners so they can tailor to their specific needs and policies. - -### Features - -- [4023](https://github.com/nginxinc/kubernetes-ingress/pull/4023) Read Prometheus key/cert from memory. -- [4080](https://github.com/nginxinc/kubernetes-ingress/pull/4080) Expose Location Zones metrics. -- [4127](https://github.com/nginxinc/kubernetes-ingress/pull/4127), [4200](https://github.com/nginxinc/kubernetes-ingress/pull/4200), [4223](https://github.com/nginxinc/kubernetes-ingress/pull/4223) Add path-regex annotation for ingress. -- [4108](https://github.com/nginxinc/kubernetes-ingress/pull/4108) Add command line argument for custom TLS Passthrough port. -- [4271](https://github.com/nginxinc/kubernetes-ingress/pull/4271) Add custom listener controls to VirtualServer. - -### Fixes - -- [4160](https://github.com/nginxinc/kubernetes-ingress/pull/4160) Update JWT/JWKS policy validation. -- [4371](https://github.com/nginxinc/kubernetes-ingress/pull/4371) Improve runtime batch reloads. - -### Helm Chart - -- [3977](https://github.com/nginxinc/kubernetes-ingress/pull/3977) Add support for controller.selectorLabels. Thanks to [Youqing Han](https://github.com/hanyouqing). -- [4058](https://github.com/nginxinc/kubernetes-ingress/pull/4058) Add clusterIP to service if specified in values. Thanks to [EutiziStefano](https://github.com/EutiziStefano). -- [4252](https://github.com/nginxinc/kubernetes-ingress/pull/4252) Make containerPort and hostPort customizable. -- [4331](https://github.com/nginxinc/kubernetes-ingress/pull/4331) Expose Prometheus metrics through a headless Service. -- [4351](https://github.com/nginxinc/kubernetes-ingress/pull/4351) Update helm values file to move controller.serviceMonitor to prometheus.serviceMonitor. -- [4333](https://github.com/nginxinc/kubernetes-ingress/pull/4333) Allow installing IC without creating a new ingress class. - -### Upgrade - -- For NGINX, use the 3.3.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.3.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). -- For NGINX Plus, use the 3.3.0 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.3.0 source code -- For Helm, use version 1.0.0 of the chart. - -### Supported Platforms - -We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.22-1.28. - -
- -## 3.2.1 +## NGINX Ingress Controller 3.2.1 17 Aug 2023 -### Dependencies +CHANGES: - Update NGINX version to 1.25.2. - Update NGINX Plus version to R30. - Update Go to 1.21 and Go dependencies. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.18.1. + +UPGRADE: - For NGINX, use the 3.2.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.2.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.2.1 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.2.1 source code - For Helm, use version 0.18.1 of the chart. -
- -## 3.2.0 +## NGINX Ingress Controller 3.2.0 27 June 2023 -### Features +FEATURES: - [3790](https://github.com/nginxinc/kubernetes-ingress/pull/3790) Gunzip for VS - [3863](https://github.com/nginxinc/kubernetes-ingress/pull/3863) OIDC - relaxed OIDC scope validation @@ -158,7 +42,7 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [4022](https://github.com/nginxinc/kubernetes-ingress/pull/4022) Add document to tutorial section for configuring the default OIDC implementation. - [4031](https://github.com/nginxinc/kubernetes-ingress/pull/4031) Add NGINX Plus Alpine image with FIPS inside for supported (paid) customers. -### Fixes +FIXES: - [3737](https://github.com/nginxinc/kubernetes-ingress/pull/3737) Update VirtualServer to ignore CRL for EgressMTLS. - [3798](https://github.com/nginxinc/kubernetes-ingress/pull/3798) Update VirtualServer template to generate an internal jwt auth location per policy applied. @@ -166,35 +50,32 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [3870](https://github.com/nginxinc/kubernetes-ingress/pull/3870) Add Funcs() method to UpdateVirtualServerTemplate method. Thanks to [Bryan Hendryx](https://github.com/coolbry95). - [3933](https://github.com/nginxinc/kubernetes-ingress/pull/3933) fix --external-service flag when using serviceNameOverride. Thanks to [Tim N](https://github.com/timnee). -### Dependencies +CHANGES: - Update NGINX version to 1.25.1. - Update Debian to 12 for NGINX Plus images (except for images containing the NGINX App Protect modules). - Update Alpine to 3.18 for NGINX Plus images. -### Helm Chart +HELM CHART: - [3814](https://github.com/nginxinc/kubernetes-ingress/pull/3814) Remove semverCompare for allocateLoadBalancerNodePorts. Thanks to [Alex Wied](https://github.com/centromere). - [3905](https://github.com/nginxinc/kubernetes-ingress/pull/3905) Reverse order of NAPDOS maxDaemons and maxWorkers in Helm chart. -### Upgrade +UPGRADE: - For NGINX, use the 3.2.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.2.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.2.0 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.2.0 source code. - For Helm, use version 0.18.0 of the chart. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.22-1.27. -
- -## 3.1.1 +## NGINX Ingress Controller 3.1.1 04 May 2023 -### Overview - +OVERVIEW: This release reverts the changes made in 3.1.0 to use sysctls to bind to lower level ports without the NET_BIND_SERVICE capability. It also adds support for serviceNameOverride in the Helm chart, that can be used to override the service name for the NGINX Ingress Controller. This is useful especially during an upgrade from versions prior to 3.1.0, to avoid downtime due to the service name change. To use this feature, set the `serviceNameOverride` value in the Helm chart to the name of the existing service. For example, if the existing service name is `my-release-nginx-ingress`, you can use `--set serviceNameOverride=my-release-nginx-ingress` when running the upgrade command. @@ -204,22 +85,22 @@ Here is an example upgrade command that keeps the existing service name `my-rele helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.17.1 --set serviceNameOverride=my-release-nginx-ingress ``` -### Fixes +FIXES: - [3737](https://github.com/nginxinc/kubernetes-ingress/pull/3737) Update VirtualServer to ignore CRL for EgressMTLS. - [3722](https://github.com/nginxinc/kubernetes-ingress/pull/3722) Inherit NET_BIND_SERVICE from IC to Nginx. Thanks to [Valters Jansons](https://github.com/sigv). - [3798](https://github.com/nginxinc/kubernetes-ingress/pull/3798) Update VirtualServer template to generate an internal jwt auth location per policy applied. -### Features +IMPROVEMENTS: - [3491](https://github.com/nginxinc/kubernetes-ingress/pull/3491) Egress via Ingress VirtualServer Resource. -### Dependencies +CHANGES: - Update NGINX version to 1.23.4. - Update NGINX Plus version to R29. -### Helm Chart +HELM CHART: - [3602](https://github.com/nginxinc/kubernetes-ingress/pull/3602) Updated NGINX Service Mesh references in Helm templates. Thanks to [Jared Byers](https://github.com/jbyers19). - [3773](https://github.com/nginxinc/kubernetes-ingress/pull/3773) Swap cpu and memory in HPA template. Thanks to [Bryan Hendryx](https://github.com/coolbry95). @@ -227,19 +108,17 @@ helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0. - [3815](https://github.com/nginxinc/kubernetes-ingress/pull/3815) Fix GlobalConfiguration name in Helm Chart. - [3862](https://github.com/nginxinc/kubernetes-ingress/pull/3862) Add correct indentation to controller-leader-election configmap helm template. -### Upgrade +UPGRADE: - For NGINX, use the 3.1.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.1.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.1.1 images from the F5 Container registry or build your own image using the 3.1.1 source code. - For Helm, use version 0.17.1 of the chart. -
- -## 3.1.0 +## NGINX Ingress Controller 3.1.0 29 Mar 2023 -### Overview +OVERVIEW: - Beginning with release 3.1.0 the NET_BIND_SERVICE capability is no longer used, and instead relies on net.ipv4.ip_unprivileged_port_start sysctl to allow port binding. Kubernetes 1.22 or later is required for this sysctl to be [classified as safe](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#safe-and-unsafe-sysctls). **Ensure that you are using the latest updated `deployment` and `daemonset` example yaml files available in the repo.** - *The minimum supported version of Kubernetes is now 1.22*. The NGINX Ingress Controller now uses `sysctls` to [bind to lower level ports without additional privileges](https://github.com/nginxinc/kubernetes-ingress/pull/3573/). This removes the need to use `NET_BIND_SERVICE` to bind to these ports. Thanks to [Valters Jansons](https://github.com/sigv) for making this feature possible! @@ -251,7 +130,7 @@ helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0. - The NGINX Ingress Controller uses the latest OIDC reference implementation which now supports [forwarding access tokens to upstreams / backends](https://github.com/nginxinc/kubernetes-ingress/pull/3474). Thanks to [Shawn Kim](https://github.com/shawnhankim) for making this possible! - The default TLS secret is now optional. This improves the security posture of NGINX Ingress Controller through enabling [ssl_reject_handshake](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_reject_handshake). This has the impact of immediately terminating the SSL handshake and not revealing TLS or cypher settings to calls that do not match a configured hostname. -### Features +FEATURES: - [3034](https://github.com/nginxinc/kubernetes-ingress/pull/3034) Allow extra args to be provided to the OIDC auth endpoint. Thanks to [Alan Wilkie](https://github.com/alanwilkie-finocomp). - [3474](https://github.com/nginxinc/kubernetes-ingress/pull/3474) Add access token support in the OIDC. Thanks to [Shawn Kim](https://github.com/shawnhankim). @@ -262,17 +141,20 @@ helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0. - [3451](https://github.com/nginxinc/kubernetes-ingress/pull/3451) Enable keepalive-time for healthchecks in VS and VSR. - [3560](https://github.com/nginxinc/kubernetes-ingress/pull/3560) Add support for load a pre-compiles AppProtect Policy Bundle. - [3632](https://github.com/nginxinc/kubernetes-ingress/pull/3632) Update nginx.org/ca secret type & crl field to IngressMTLS to support CRL. + +IMPROVEMENTS: + - [3629](https://github.com/nginxinc/kubernetes-ingress/pull/3629) Use the "runtime default" seccomp profile. Thanks to [Valters Jansons](https://github.com/sigv). - [3573](https://github.com/nginxinc/kubernetes-ingress/pull/3573) Rework port binding logic without privileges. Thanks to [Valters Jansons](https://github.com/sigv). - [3646](https://github.com/nginxinc/kubernetes-ingress/pull/3646) Remove app protect agent. - [3507](https://github.com/nginxinc/kubernetes-ingress/pull/3507) Support empty path for ImplementationSpecific pathType. - [3482](https://github.com/nginxinc/kubernetes-ingress/pull/3482) Use new NSM Spiffe and Cert rotation library. - [3442](https://github.com/nginxinc/kubernetes-ingress/pull/3442) Add websocket protocol option to monitor directive. -- [3674](https://github.com/nginxinc/kubernetes-ingress/pull/3674) Move NAP DoS chart to new repo. +- [3674](https://github.com/nginxinc/kubernetes-ingress/pull/3674) Move NAP DoS chart to new repo - [3302](https://github.com/nginxinc/kubernetes-ingress/pull/3302) Make default-server-secret optional. -- [3586](https://github.com/nginxinc/kubernetes-ingress/pull/3586) Add new labels and metadata to add version information to pods. +- [3586](https://github.com/nginxinc/kubernetes-ingress/pull/3586) Add new labels and metadata to add version information to pods -### Fixes +FIXES: - [3463](https://github.com/nginxinc/kubernetes-ingress/pull/3463) Support non-vs created Challenge Ingress. - [3475](https://github.com/nginxinc/kubernetes-ingress/pull/3475) Ensure leader election is correctly disabled when option is set to `false` in helm template. @@ -282,7 +164,7 @@ helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0. - [3607](https://github.com/nginxinc/kubernetes-ingress/pull/3607) Clear Content-Length headers for requests processed by internal JWKS routes. - [3660](https://github.com/nginxinc/kubernetes-ingress/pull/3660) Remove unwanted chars from label value. -### Helm Chart +HELM CHART: - [3581](https://github.com/nginxinc/kubernetes-ingress/pull/3581) Push edge Helm Chart to OCI registries. - [3449](https://github.com/nginxinc/kubernetes-ingress/pull/3449) Correct values.schema.json nodeSelector. @@ -291,61 +173,63 @@ helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0. - [3537](https://github.com/nginxinc/kubernetes-ingress/pull/3537) Update schema references to k8s v1.26.1. - [3606](https://github.com/nginxinc/kubernetes-ingress/pull/3606) Fix Helm Chart labels and templates. Move version update to labels. -### Upgrade +UPGRADE: - Make sure the Kubernetes version is in the supported platforms listed below. - For NGINX, use the 3.1.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.1.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.1.0 images from the F5 Container registry or build your own image using the 3.1.0 source code. - For Helm, use version 0.17.0 of the chart. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.22-1.26. -
- -## 3.0.2 +## NGINX Ingress Controller 3.0.2 13 Feb 2023 -### Fixes +FIXES: - [3519](https://github.com/nginxinc/kubernetes-ingress/pull/3519) Add OnDelete to allowed strategy values - [3541](https://github.com/nginxinc/kubernetes-ingress/pull/3541) Ensure non-ready endpoints are not added to upstreams - [3527](https://github.com/nginxinc/kubernetes-ingress/pull/3527) Fix controller.topologySpreadConstraints schema, thanks to [Marco Londero](https://github.com/marcuz) -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.16.2. + +UPGRADE: - For NGINX, use the 3.0.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.0.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.0.2 images from the F5 Container registry or build your own image using the 3.0.2 source code. - For Helm, use version 0.16.2 of the chart. -
- -## 3.0.1 +## NGINX Ingress Controller 3.0.1 25 Jan 2023 -### Fixes +FIXES: - [3448](https://github.com/nginxinc/kubernetes-ingress/pull/3448) Fix Helm Chart Schema for priorityClassName - [3449](https://github.com/nginxinc/kubernetes-ingress/pull/3449) Correct nodeSelector in the Helm Chart Schema - [3463](https://github.com/nginxinc/kubernetes-ingress/pull/3463) Support non-VS created Challenge Ingress - [3481](https://github.com/nginxinc/kubernetes-ingress/pull/3481) Add missing OSS internal routes for NGINX Service Mesh -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.16.1. + +UPGRADE: - For NGINX, use the 3.0.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.0.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.0.1 images from the F5 Container registry or build your own image using the 3.0.1 source code. - For Helm, use version 0.16.1 of the chart. -
- -## 3.0.0 +## NGINX Ingress Controller 3.0.0 12 January 2023 -### Overview +OVERVIEW: - Added support for [Deep Service Insight](https://docs.nginx.com/nginx-ingress-controller/logging-and-monitoring/service-insight) for VirtualServer and TransportServer using the [-enable-service-insight](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-enable-service-insight) cli argument. - *The minimum supported version of Kubernetes is now 1.21*. NGINX Ingress Controller 3.0.0 removes support for `k8s.io/v1/Endpoints` API in favor of `discovery.k8s.io/v1/EndpointSlices`. For older Kubernetes versions, use the 2.4.x release of the Ingress Controller. @@ -357,18 +241,18 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - Beginning with NGINX Service Mesh release 1.7 it will include support for the free version of NGINX Ingress Controller as well as the paid version. - NGINX Ingress Controller + NGINX App Protect Denial of Service is now available through the AWS Marketplace. -### Breaking Changes +FEATURES: - [3260](https://github.com/nginxinc/kubernetes-ingress/pull/3260) Added support for EndpointSlices. - -### Features - - [3299](https://github.com/nginxinc/kubernetes-ingress/pull/3299) Support Dynamic namespaces using Labels. - [3261](https://github.com/nginxinc/kubernetes-ingress/pull/3261) Deep service insight endpoint for VirtualServer CR. - [3361](https://github.com/nginxinc/kubernetes-ingress/pull/3361) Added healthcheck for TransportServer CR. - [3347](https://github.com/nginxinc/kubernetes-ingress/pull/3347) Import JWKS from URL on JWT policy. - [3274](https://github.com/nginxinc/kubernetes-ingress/pull/3274) Allow configuration of map-hash-bucket-size and map-hash-max-size directives. - [3376](https://github.com/nginxinc/kubernetes-ingress/pull/3376) NGINX Service Mesh will support the free version of NGINX Ingress Controller when using NGINX open source. + +IMPROVEMENTS: + - [3170](https://github.com/nginxinc/kubernetes-ingress/pull/3170) Watch subset of namespaces for secrets. Thanks to [Hans Feldt](https://github.com/hafe). - [3341](https://github.com/nginxinc/kubernetes-ingress/pull/3341) Set value of `$remote_addr` to client IP when TLSPassthrough and Proxy Protocol are enabled. - [3131](https://github.com/nginxinc/kubernetes-ingress/pull/3131) NAP DoS images are now available in the AWS Marketplace. @@ -376,11 +260,11 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [2735](https://github.com/nginxinc/kubernetes-ingress/pull/2735) Support default client proxy headers to be overwritten in VirtualServer. Thanks to [Alex Wied](https://github.com/centromere). - [3133](https://github.com/nginxinc/kubernetes-ingress/pull/3133) Added caseSensitiveHttpHeaders to APPolicy CRD. Thanks to [Pavel Galitskiy](https://github.com/galitskiy). -### Fixes +FIXES: - [3139](https://github.com/nginxinc/kubernetes-ingress/pull/3139) Remove all IPV6 listeners in ingress resources with `-disable-ipv6` command line. -### Helm Chart +HELM CHART: - [3113](https://github.com/nginxinc/kubernetes-ingress/pull/3113) Added JSON Schema. - [3143](https://github.com/nginxinc/kubernetes-ingress/pull/3143) Added annotations for deployment and daemonset. @@ -388,83 +272,87 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [3065](https://github.com/nginxinc/kubernetes-ingress/pull/3065) Added annotations to the service account. Thanks to [0m1xa](https://github.com/0m1xa). - [3276](https://github.com/nginxinc/kubernetes-ingress/pull/3276) Added horizontalpodautoscaler. Thanks to [Bryan Hendryx](https://github.com/coolbry95). -### Upgrade +UPGRADE: - Make sure the Kubernetes version is in the supported platforms listed below. - For NGINX, use the 3.0.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.0.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 3.0.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 3.0.0 source code. - For Helm, use version 0.16.0 of the chart. Helm does not upgrade the CRDs. If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.21-1.26. -
- -## 2.4.2 +## NGINX Ingress Controller 2.4.2 30 Nov 2022 -### Dependencies +CHANGES: - Update NGINX Plus version to R28. - Update NGINX App Protect WAF version to 4.0. - Update NGINX App Protect DoS version to 3.1. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.15.2. + +UPGRADE: - For NGINX, use the 2.4.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.4.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 2.4.2 images from the F5 Container registry or build your own image using the 2.4.2 source code. - For Helm, use version 0.15.2 of the chart. -
- -## 2.4.1 +## NGINX Ingress Controller 2.4.1 19 October 2022 -### Dependencies +CHANGES: - [3183](https://github.com/nginxinc/kubernetes-ingress/pull/3183) Update NGINX version to 1.23.2. - [3175](https://github.com/nginxinc/kubernetes-ingress/pull/3175) Update Go dependencies. - Update NGINX Plus version to R27 P1. -### Fixes +FIXES: - [3139](https://github.com/nginxinc/kubernetes-ingress/pull/3139) Remove all IPV6 listeners in ingress resources with -disable-ipv6 command line. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.15.1. + +UPGRADE: - For NGINX, use the 2.4.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.4.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 2.4.1 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.4.1 source code. - For Helm, use version 0.15.1 of the chart. -
- -## 1.12.5 +## NGINX Ingress Controller 1.12.5 19 October 2022 -### Dependencies +CHANGES: - Update NGINX version to 1.23.2. - Update NGINX Plus version to R27 P1. - Update Alpine to 3.16. - Update Go to 1.19 and Go dependencies. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.10.5. + +UPGRADE: - For NGINX, use the 1.12.5 image from our DockerHub: `nginx/nginx-ingress:1.12.5`, `nginx/nginx-ingress:1.12.5-alpine` or `nginx/nginx-ingress:1.12.5-ubi` - For NGINX Plus, please build your own image using the 1.12.5 source code. - For Helm, use version 0.10.5 of the chart. -
- -## 2.4.0 +## NGINX Ingress Controller 2.4.0 04 October 2022 -### Overview +OVERVIEW: - Added support for enabling [proxy_protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v2.4.0/examples/shared-examples/proxy-protocol) when port 443 is being used for both HTTPS traffic and [TLS Passthrough traffic](https://github.com/nginxinc/kubernetes-ingress/tree/v2.4.0/examples/custom-resources/tls-passthrough). - Updates to the TransportServer resource to support using [ExternalName services](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). For examples, see [externalname-services](https://github.com/nginxinc/kubernetes-ingress/tree/v2.4.0/examples/custom-resources/externalname-services). @@ -474,7 +362,7 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - A new cli argument has been added: [-include-year](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-include-year). This appends the current year to the log output from the Ingress Controller. Example output: `I20220512 09:20:42.345457`. - Post-startup configuration reloads have been optimized to reduce traffic impacts. When many resources are modified at the same time, changes are combined to reduce the number of data plane reloads. -### Features +FEATURES: - [2986](https://github.com/nginxinc/kubernetes-ingress/pull/2986) Batch reloads at runtime. - [2914](https://github.com/nginxinc/kubernetes-ingress/pull/2914) Support watching multiple namespaces. @@ -482,15 +370,18 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [2993](https://github.com/nginxinc/kubernetes-ingress/pull/2993) Accept proxy protocol when TLS passthrough enabled. - [3041](https://github.com/nginxinc/kubernetes-ingress/pull/3041) Support external name service for TransportServer. - [2939](https://github.com/nginxinc/kubernetes-ingress/pull/2939) Add support for wildcard hostname in VirtualServer. + +IMPROVEMENTS: + - [3040](https://github.com/nginxinc/kubernetes-ingress/pull/3040) Add command line argument to manually disable IPV6 listeners for unsupported clusters. - [3088](https://github.com/nginxinc/kubernetes-ingress/pull/3088) Filter secrets of type helm.sh/release.v1. -### Fixes +FIXES: - [2971](https://github.com/nginxinc/kubernetes-ingress/pull/2971) fix: Correct error message on missing path in path validation. Thanks to [Zachary Seguin](https://github.com/zachomedia). - [3095](https://github.com/nginxinc/kubernetes-ingress/pull/3095) do not create configmap if customConfigMap is used. Thanks to [Bryan Hendryx](https://github.com/coolbry95). -### Helm Chart +HELM CHART: - [3087](https://github.com/nginxinc/kubernetes-ingress/pull/3087) Allow omitting the default server secret from Helm installs. - [2831](https://github.com/nginxinc/kubernetes-ingress/pull/2831) Add ServiceMonitor to Helm Chart. Thanks to [araineUnity](https://github.com/araineUnity). @@ -498,51 +389,48 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [2979](https://github.com/nginxinc/kubernetes-ingress/pull/2979) Allow to specify image with digest in helm chart. Thanks to [Hans Feldt](https://github.com/hafe). - [3031](https://github.com/nginxinc/kubernetes-ingress/pull/3031) Adding automountServiceAccountToken to helm chart. -### Upgrade +UPGRADE: - For NGINX, use the 2.4.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.4.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.4.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.4.0 source code. - For Helm, use version 0.15.0 of the chart. If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs will not be upgraded. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.25. -
- -## 2.3.1 +## NGINX Ingress Controller 2.3.1 16 September 2022 -### Dependencies +CHANGES: - [3048](https://github.com/nginxinc/kubernetes-ingress/pull/3048) Bump NGINX to 1.23.1 - [3049](https://github.com/nginxinc/kubernetes-ingress/pull/3049) Update Go dependencies. +- [3031](https://github.com/nginxinc/kubernetes-ingress/pull/3031) Add automountServiceAccountToken to helm chart -### Helm Chart +HELM CHART: -- [3031](https://github.com/nginxinc/kubernetes-ingress/pull/3031) Add automountServiceAccountToken to helm chart +- The version of the Helm chart is now 0.14.1. -### Upgrade +UPGRADE: - For NGINX, use the 2.3.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.3.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 2.3.1 images from the F5 Container registry or build your own image using the 2.3.1 source code. - For Helm, use version 0.14.1 of the chart. -
- -## 2.3.0 +## NGINX Ingress Controller 2.3.0 12 July 2022 -### Overview +OVERVIEW: - Support making VirtualServer resources discoverable via public DNS servers using [external-dns](https://kubernetes-sigs.github.io/external-dns). Examples for configuring external-dns with NGINX Ingress Controller can be found [here](https://github.com/nginxinc/kubernetes-ingress/tree/v2.3.0/examples/custom-resources/external-dns). - Resolves [CVE-2022-30535](https://support.f5.com/csp/article/K52125139). This vulnerability impacted the visibility of secrets accessible by NGINX Ingress Controller. In some cases, secrets visible to NGINX Ingress Controller could be exposed to any authenticated user with permission to create and update Ingress objects. This vulnerability affected Ingress objects only - our Custom Resources (VirtualServer and TransportServer) were not affected. Customers unable to upgrade should migrate any Ingress resources to VirtualServer resources where possible, and use RBAC to restrict write access for users for Ingress objects. - Support using HTTP basic authentication with [VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v2.3.1/examples/custom-resources/basic-auth) and [Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v2.3.0/examples/basic-auth) resources. Special thanks to [Simon Wachter](https://github.com/svvac). - Support HTTP01 type ACME Issuers for use with VirtualServer resources with [cert-manager](https://cert-manager.io/docs/). -### Features +FEATURES: - [2581](https://github.com/nginxinc/kubernetes-ingress/pull/2581) Add OpenTracing to all Debian and Alpine based images. - [2328](https://github.com/nginxinc/kubernetes-ingress/pull/2328) Add handling of multiple log destinations. @@ -552,6 +440,9 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [2801](https://github.com/nginxinc/kubernetes-ingress/pull/2801) Add SBOMs to release. - [2269](https://github.com/nginxinc/kubernetes-ingress/pull/2269) HTTP basic auth support. Thanks to [Simon Wachter](https://github.com/svvac). - [2800](https://github.com/nginxinc/kubernetes-ingress/pull/2800) Integrate external-dns with VirtualServer resources. + +IMPROVEMENTS: + - [2583](https://github.com/nginxinc/kubernetes-ingress/pull/2583) Add runAsNonRoot in deployments. - [2484](https://github.com/nginxinc/kubernetes-ingress/pull/2484) Add container resource requests. - [2627](https://github.com/nginxinc/kubernetes-ingress/pull/2627) Update InternalRoute server_name. @@ -566,7 +457,7 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [2783](https://github.com/nginxinc/kubernetes-ingress/pull/2783) Update validation regex for path spec. - [2781](https://github.com/nginxinc/kubernetes-ingress/pull/2781) Report Hostname in ExternalEndpoint for VS and VSR resources. -### Fixes +FIXES: - [2617](https://github.com/nginxinc/kubernetes-ingress/pull/2617) Fix Dockerfile for amd64 microarchitectures. - [2637](https://github.com/nginxinc/kubernetes-ingress/pull/2637) Add terminationGracePeriodSeconds to deployment. Thanks to [Maksym Iv](https://github.com/maksym-iv). @@ -574,66 +465,68 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes - [2673](https://github.com/nginxinc/kubernetes-ingress/pull/2673) Fix status.loadbalancer.hostname deletion on OOMKill. Thanks to [Heiko Voigt](https://github.com/hvoigt). - [2718](https://github.com/nginxinc/kubernetes-ingress/pull/2718) Fix cases where CM enabled but no TLS block specified in VS. -### Helm Chart +HELM CHART: - [2418](https://github.com/nginxinc/kubernetes-ingress/pull/2418) Add support for allocateLoadBalancerNodePorts, ipFamilyPolicy and ipFamilies. Thanks to [centromere](https://github.com/centromere). - [2672](https://github.com/nginxinc/kubernetes-ingress/pull/2672) Add minReadySeconds & strategy support. Thanks to [Ciaran](https://github.com/cmk-pcs). - [2625](https://github.com/nginxinc/kubernetes-ingress/pull/2625) allow configuring topologySpreadConstraints in Helm chart. Thanks to [Kamil Domański](https://github.com/kdomanski). -### Upgrade +UPGRADE: - For NGINX, use the 2.3.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.3.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.3.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.3.0 source code. - For Helm, use version 0.14.0 of the chart. If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs will not be upgraded. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. - When upgrading using the [manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/), make sure to update the [ClusterRole](https://github.com/nginxinc/kubernetes-ingress/blob/v2.3.1/deployments/rbac/rbac.yaml). This is required to enable the ExternalDNS for VirtualServer resources integration. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.24. -
- -## 2.2.2 +## NGINX Ingress Controller 2.2.2 23 May 2022 -### Fixes +FIXES: - [2627](https://github.com/nginxinc/kubernetes-ingress/pull/2627) Update InternalRoute server_name. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.13.2. + +UPGRADE: - For NGINX, use the 2.2.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.2.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.2.2 images from the F5 Container registry or build your own image using the 2.2.2 source code. - For Helm, use version 0.13.2 of the chart. -
- -## 2.2.1 +## NGINX Ingress Controller 2.2.1 17 May 2022 -### Dependencies +CHANGES: - Update Go dependencies. -### Fixes +FIXES: - [2654](https://github.com/nginxinc/kubernetes-ingress/pull/2654) Sync changes from [nginx-openid-connect](https://github.com/nginxinc/nginx-openid-connect) repo, add zoneSyncLeeway field in policy. For more information on the fixes, see [52](https://github.com/nginxinc/nginx-openid-connect/pull/52). -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.13.1. + +UPGRADE: - For NGINX, use the 2.2.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.2.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.2.1 images from the F5 Container registry or build your own image using the 2.2.1 source code. - For Helm, use version 0.13.1 of the chart. -
- -## 2.2.0 +## NGINX Ingress Controller 2.2.0 12 April 2022 -### Overview +OVERVIEW: - Support for automatic provisioning and management of Certificate resources for VirtualServer resources using [cert-manager](https://cert-manager.io/docs/). Examples for configuring cert-manager with NGINX Ingress Controller can be found [here](https://github.com/nginxinc/kubernetes-ingress/tree/v2.2.0/examples/custom-resources/certmanager). Please note that HTTP01 type ACME Issuers are not yet supported for use with VirtualServer resources. @@ -643,30 +536,36 @@ We will provide technical support for the NGINX Ingress Controller on any Kubern - A new [-enable-oidc](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-enable-oidc) cli argument has been added to enable OIDC policies. Previously, this behaviour was achieved through the usage of the `-enable-preview-policies` cli argument. -### Features +FEATURES: - [2576](https://github.com/nginxinc/kubernetes-ingress/pull/2576) Add support for IPv6. - [2572](https://github.com/nginxinc/kubernetes-ingress/pull/2572) Automate provisioning of Certificate resources for VirtualServer resources using cert-manager. + +IMPROVEMENTS: + - [2346](https://github.com/nginxinc/kubernetes-ingress/pull/2346) Use os.ReadDir for lightweight directory reading. Thanks to [Eng Zer Jun](https://github.com/Juneezee). - [2360](https://github.com/nginxinc/kubernetes-ingress/pull/2360) Add NGINX App Protect reconnect period directive. - [2479](https://github.com/nginxinc/kubernetes-ingress/pull/2479) Add cli argument to configure NGINX App Protect log level. - [2455](https://github.com/nginxinc/kubernetes-ingress/pull/2455) Increase memory available for NGINX App Protect xml parser. + +CHANGES: + - [2580](https://github.com/nginxinc/kubernetes-ingress/pull/2580) Create -enable-oidc command line argument for OIDC policy. - [2566](https://github.com/nginxinc/kubernetes-ingress/pull/2566) Unbind policy from preview policies. - [2582](https://github.com/nginxinc/kubernetes-ingress/pull/2582) Rename Make targets from `openshift` to `ubi`. -### Fixes +FIXES: - [2378](https://github.com/nginxinc/kubernetes-ingress/pull/2378) Fix healthcheck ports. - [2404](https://github.com/nginxinc/kubernetes-ingress/pull/2404) Start nginx with -e stderr parameter. - [2414](https://github.com/nginxinc/kubernetes-ingress/pull/2414) Fix in file nginx-plus.virtualserver.tmpl ApDosMonitor->ApDosMonitorURI. -### Helm Chart +HELM CHART: - [2525](https://github.com/nginxinc/kubernetes-ingress/pull/2525) Extend helm chart to include NGINX Service Mesh fields. - [2294](https://github.com/nginxinc/kubernetes-ingress/pull/2294) Add extra containers to helm chart. Thanks to [Márk Sági-Kazár](https://github.com/sagikazarmark). -### Upgrade +UPGRADE: - For NGINX, use the 2.2.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.2.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.2.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.2.0 source code. @@ -675,33 +574,33 @@ We will provide technical support for the NGINX Ingress Controller on any Kubern - The -enable-preview-policies cli argument has been deprecated, and is no longer required for any Policy resources. - Enabling OIDC Policies now requires the use of -enable-oidc cli argument instead of the -enable-preview-policies cli argument. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.23. -
- -## 2.1.2 +## NGINX Ingress Controller 2.1.2 29 March 2022 -### Dependencies +CHANGES: - Update UBI based images to 8. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.12.2. + +UPGRADE: - For NGINX, use the 2.1.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.1.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.1.2 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.1.2 source code. - For Helm, use version 0.12.2 of the chart. -
- -## 1.12.4 +## NGINX Ingress Controller 1.12.4 23 March 2022 -### Dependencies +CHANGES: - Update NGINX version to 1.21.6. - Update NGINX Plus version to R26. @@ -710,40 +609,44 @@ We will provide technical support for the NGINX Ingress Controller on any Kubern - Update UBI to 8. - Update Go to 1.17 and Go dependencies. -### Fixes +FIXES: - Fix OpenTracing not working with NGINX Plus. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.10.4. + +UPGRADE: - For NGINX, use the 1.12.4 image from our DockerHub: `nginx/nginx-ingress:1.12.4`, `nginx/nginx-ingress:1.12.4-alpine` or `nginx/nginx-ingress:1.12.4-ubi` - For NGINX Plus, please build your own image using the 1.12.4 source code. - For Helm, use version 0.10.4 of the chart. -
- -## 2.1.1 +## NGINX Ingress Controller 2.1.1 17 February 2022 -### Dependencies +CHANGES: - Update NGINX version to 1.21.6. - Update NGINX Plus version to R26. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.12.1. + +UPGRADE: - For NGINX, use the 2.1.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.1.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.1.1 images from the F5 Container registry or build your own image using the 2.1.1 source code. - For Helm, use version 0.12.1 of the chart. -
- -## 2.1.0 +## NGINX Ingress Controller 2.1.0 06 January 2022 -### Overview +OVERVIEW: - Support for NGINX App Protect Denial of Service protection with NGINX Ingress Controller. More information about [NGINX App Protect DoS](https://www.nginx.com/products/nginx-app-protect/denial-of-service/). Examples for configuring NGINX App Protect DoS with NGINX Ingress Controller can be found [here](https://github.com/nginxinc/kubernetes-ingress/tree/v2.1.1/examples/appprotect-dos). @@ -754,7 +657,7 @@ Mandatory health checks can be marked as persistent, so that the previous state See the settings [here](https://docs.nginx.com/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#upstreamhealthcheck). More about the [NGINX Plus mandatory and persistent health check features](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#mandatory-health-checks) -### Features +FEATURES: - [2251](https://github.com/nginxinc/kubernetes-ingress/pull/2251) Enable setting mandatory and persistent in upstream healthchecks in VS and VSR. - [2241](https://github.com/nginxinc/kubernetes-ingress/pull/2241) Add support for NGINX App Protect DoS. @@ -765,14 +668,16 @@ More about the [NGINX Plus mandatory and persistent health check features](https - [2100](https://github.com/nginxinc/kubernetes-ingress/pull/2100) Add support for initContainers. Thanks to [Gunnar Scherf](https://github.com/g10f). - [1827](https://github.com/nginxinc/kubernetes-ingress/pull/1827) Add support for wildcard cert in VirtualServer resources. Thanks to [Simon Wachter](https://github.com/svvac). - [2107](https://github.com/nginxinc/kubernetes-ingress/pull/2107) Add option to download the NGINX Ingress Controller binary. Introduced a new `TARGET` `download` in the `Makefile` which can be used when building the NGINX Ingress Controller Docker image. With this option the Ingress Controller binary will be downloaded instead of built from source. + +IMPROVEMENTS: + - [2044](https://github.com/nginxinc/kubernetes-ingress/pull/2044) Upload NGINX Ingress Controller binaries to release. - [2094](https://github.com/nginxinc/kubernetes-ingress/pull/2094) AP: update appolicies crd. - [2216](https://github.com/nginxinc/kubernetes-ingress/pull/2216) Add grpc_status to the logs. - [2237](https://github.com/nginxinc/kubernetes-ingress/pull/2237) Unbind app-protect from -preview-policies. -- [2273](https://github.com/nginxinc/kubernetes-ingress/pull/2273) Make the resource comparison more informative in case of an error. Thanks to [Andrey Karpov](https://github.com/ndk). -- [2124](https://github.com/nginxinc/kubernetes-ingress/pull/2124) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. +- [2273](https://github.com/nginxinc/kubernetes-ingress/pull/2273) Make the resource comparison more informative in case of an error. Thanks to [Andrey Karpov](https://github.com/ndk) -### Fixes +FIXES: - [2267](https://github.com/nginxinc/kubernetes-ingress/pull/2267) Fix URI rewrite in VirtualServers and VirtualServerRoutes. - [2260](https://github.com/nginxinc/kubernetes-ingress/pull/2260) Check if refresh token is `undefined` and do not store it in this case. Thanks to [tippexs](https://github.com/tippexs) for the fix. @@ -781,122 +686,134 @@ More about the [NGINX Plus mandatory and persistent health check features](https - [2125](https://github.com/nginxinc/kubernetes-ingress/pull/2125) Allow empty string in server-tokens annotation for NGINX Plus. - [2042](https://github.com/nginxinc/kubernetes-ingress/pull/2042) Use release specific repo for NGINX Plus on Debian. -### Dependencies +CHANGES: +- [2124](https://github.com/nginxinc/kubernetes-ingress/pull/2124) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. - [2173](https://github.com/nginxinc/kubernetes-ingress/pull/2173) Update Debian to Bullseye. - Update NGINX Plus version to R25. - Update NGINX version to 1.21.5. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.12.0. + +UPGRADE: - For NGINX, use the 2.1.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.1.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.1.0 images from the F5 Container registry or build your own image using the 2.1.0 source code. - For Helm, use version 0.12.0 of the chart. - We changed the behaviour of snippets in Ingress resources by extending the existing -enable-snippets cli argument to apply to Ingress resources as well as VirtualServer, VirtualServerRoute and TransportServer resources. Because the default value of -enable-snippets is false, if you are using snippets in Ingress resources, you must explicitly set the -enable-snippets to true before upgrading the Ingress Controller, so that the new version of the Ingress Controller doesn't reject Ingresses with snippets annotations. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.23. -
- -## 2.0.3 +## NGINX Ingress Controller 2.0.3 28 October 2021 -### Breaking Changes +CHANGES: - [2124](https://github.com/nginxinc/kubernetes-ingress/pull/2124) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. +- [2132](https://github.com/nginxinc/kubernetes-ingress/pull/2132) Install libcurl on OpenTracing for NGINX Plus. -### Fixes +HELM CHART: -- [2132](https://github.com/nginxinc/kubernetes-ingress/pull/2132) Install libcurl on OpenTracing for NGINX Plus. +- The version of the Helm chart is now 0.11.3. -### Upgrade +UPGRADE: - For NGINX, use the 2.0.3 image from our DockerHub: `nginx/nginx-ingress:2.0.3`, `nginx/nginx-ingress:2.0.3-alpine` or `nginx/nginx-ingress:2.0.3-ubi` - For NGINX Plus, please build your own image using the 2.0.3 source code. - For Helm, use version 0.11.3 of the chart. - We changed the behaviour of snippets in Ingress resources by extending the existing -enable-snippets cli argument to apply to Ingress resources as well as VirtualServer, VirtualServerRoute and TransportServer resources. Because the default value of -enable-snippets is false, if you are using snippets in Ingress resources, you must explicitly set the -enable-snippets to true before upgrading the Ingress Controller, so that the new version of the Ingress Controller doesn't reject Ingresses with snippets annotations. -
- -## 1.12.3 +## NGINX Ingress Controller 1.12.3 28 October 2021 -### Fixes +FIXES: - [2133](https://github.com/nginxinc/kubernetes-ingress/pull/2133) Use release specific repo for the App Protect packages on Debian. This fixes an error when building Debian-based images with NGINX Plus with App Protect: previously, building an image would fail with the error `nginx-plus-module-appprotect : Depends: app-protect-plugin (= 3.639.0-1~buster) but 3.671.0-1~buster is to be installed`. The bug first appeared when NGINX App Protect version 3.6 was released on 13 October 2021. - [2134](https://github.com/nginxinc/kubernetes-ingress/pull/2134) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.10.3. + +UPGRADE: - For NGINX, use the 1.12.3 image from our DockerHub: `nginx/nginx-ingress:1.12.3`, `nginx/nginx-ingress:1.12.3-alpine` or `nginx/nginx-ingress:1.12.3-ubi` - For NGINX Plus, please build your own image using the 1.12.3 source code. - For Helm, use version 0.10.3 of the chart. - We changed the behaviour of snippets in Ingress resources by extending the existing -enable-snippets cli argument to apply to Ingress resources as well as VirtualServer, VirtualServerRoute and TransportServer resources. Because the default value of -enable-snippets is false, if you are using snippets in Ingress resources, you must explicitly set the -enable-snippets to true before upgrading the Ingress Controller, so that the new version of the Ingress Controller doesn't reject Ingresses with snippets annotations. -
- -## 2.0.2 +## NGINX Ingress Controller 2.0.2 13 October 2021 -### Dependencies +CHANGES: - Update NGINX App Protect version to 3.6. - Update NGINX Plus version to R25 in NGINX App Protect enabled images. +- [2074](https://github.com/nginxinc/kubernetes-ingress/pull/2074) Update JWT library to `golang-jwt/jwt`. Previously, the Ingress Controller used `dgrijalva/jwt-go`, which has a vulnerability [CVE-2020-26160](https://nvd.nist.gov/vuln/detail/CVE-2020-26160). Note that the Ingress Controller wasn’t affected by this vulnerability, and the jwt library was used only in the NGINX Plus images from AWS Marketplace for Containers. -### Features +HELM CHART: -- [2074](https://github.com/nginxinc/kubernetes-ingress/pull/2074) Update JWT library to `golang-jwt/jwt`. Previously, the Ingress Controller used `dgrijalva/jwt-go`, which has a vulnerability [CVE-2020-26160](https://nvd.nist.gov/vuln/detail/CVE-2020-26160). Note that the Ingress Controller wasn’t affected by this vulnerability, and the jwt library was used only in the NGINX Plus images from AWS Marketplace for Containers. +- The version of the Helm chart is now 0.11.2. -### Upgrade +UPGRADE: - For NGINX, use the 2.0.2 image from our DockerHub. - For NGINX Plus, use the 2.0.2 from the F5 Container registry or build your own image using the 2.0.2 source code. - For Helm, use version 0.11.2 of the chart. -
- -## 2.0.1 +## NGINX Ingress Controller 2.0.1 07 October 2021 -### Fixes +FIXES: - [2051](https://github.com/nginxinc/kubernetes-ingress/pull/2051) Use release specific repo for NGINX Plus on Debian. This fixes an error when building the Debian-based image with NGINX Plus and App Protect: previously, building the image would fail with the error `Package 'nginx-plus-r24' has no installation candidate`. -### Upgrade +DOCUMENTATION IMPROVEMENTS: + +- [2059](https://github.com/nginxinc/kubernetes-ingress/pull/2059) fixes issues with the 404 and robots.txts redirects. +- [2049](https://github.com/nginxinc/kubernetes-ingress/pull/2049) Remove note from operator installation. A version of the Operator compatible with 2.0.x is now available. + +HELM CHART: + +- The version of the Helm chart is now 0.11.1. + +UPGRADE: - For NGINX, use the 2.0.1 image from our DockerHub. - For NGINX Plus, use the 2.0.1 from the F5 Container registry or build your own image using the 2.0.1 source code. - For Helm, use version 0.11.1 of the chart. -
- -## 1.12.2 +## NGINX Ingress Controller 1.12.2 7 October 2021 -### Fixes +FIXES: - [2048](https://github.com/nginxinc/kubernetes-ingress/pull/2048) Use release specific repo for NGINX Plus on Debian. This fixes an error when building Debian-based images with NGINX Plus: previously, building an image would fail with the error `Package 'nginx-plus-r24' has no installation candidate`. The bug first appeared when NGINX Plus R25 was released on 28 September 2021. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.10.2. + +UPGRADE: - For NGINX, use the 1.12.2 image from our DockerHub: `nginx/nginx-ingress:1.12.2`, `nginx/nginx-ingress:1.12.2-alpine` or `nginx/nginx-ingress:1.12.2-ubi` - For NGINX Plus, please build your own image using the 1.12.2 source code. - For Helm, use version 0.10.2 of the chart. -
- -## 2.0.0 +## NGINX Ingress Controller 2.0.0 28 September 2021 -### Overview +OVERVIEW: Release 2.0.0 includes: @@ -913,26 +830,24 @@ Release 2.0.0 includes: You will find the complete changelog for release 2.0.0, including bug fixes, improvements, and changes below. -### Breaking Changes +FEATURES: +- [1908](https://github.com/nginxinc/kubernetes-ingress/pull/1908) Add NTLM support to VirtualServer and VirtualServerRoute upstreams. - [1850](https://github.com/nginxinc/kubernetes-ingress/pull/1850) Support Ingress and IngressClass v1. +- [1746](https://github.com/nginxinc/kubernetes-ingress/pull/1746) Add ingressClassName field to Policy. -### Features +IMPROVEMENTS: -- [1908](https://github.com/nginxinc/kubernetes-ingress/pull/1908) Add NTLM support to VirtualServer and VirtualServerRoute upstreams. -- [1746](https://github.com/nginxinc/kubernetes-ingress/pull/1746) Add ingressClassName field to Policy. - [1956](https://github.com/nginxinc/kubernetes-ingress/pull/1956) Add v1alpha1 version back to policy CRD. - [1907](https://github.com/nginxinc/kubernetes-ingress/pull/1907) Remove libs compilation for OpenTracing in Dockerfile; add Zipkin and Datadog in addition to the already supported Jaeger tracer; additionally, for NGINX we now publish a Docker image with the tracers and the OpenTracing module on DockerHub: `nginx-ic/nginx-plus-ingress:1.12.0-ot`. Also thanks to [MatyRi](https://github.com/MatyRi) for upgrading OpenTracing in [1883](https://github.com/nginxinc/kubernetes-ingress/pull/1883). - [1788](https://github.com/nginxinc/kubernetes-ingress/pull/1788) Reload only once during the start. This significantly reduces the time it takes for an Ingress Controller pod to become ready when hundreds of Ingress or other supported resources are created in the cluster. -- [1721](https://github.com/nginxinc/kubernetes-ingress/pull/1721) Increase default reload timeout to 60s: the Ingress Controller will wait for 60s for NGINX to start or reload. Previously, the default was 4 seconds. -- [2009](https://github.com/nginxinc/kubernetes-ingress/pull/2009) Increase default upstream zone size for NGINX Plus. See the INCREASED UPSTREAM ZONES section below. -### Fixes +FIXES: - [1926](https://github.com/nginxinc/kubernetes-ingress/pull/1926) Fix increased IC pod startup time when hundreds of VirtualServerRoutes are used - [1712](https://github.com/nginxinc/kubernetes-ingress/pull/1712) Allow `make` to build image when .git directory is missing. -### Documentation +DOCUMENTATION IMPROVEMENTS: - [1932](https://github.com/nginxinc/kubernetes-ingress/pull/1932) Add IAM instructions for NGINX Plus AWS Marketplace images. - [1927](https://github.com/nginxinc/kubernetes-ingress/pull/1927) Fix function name comments typo. Thanks to [Sven Nebel](https://github.com/snebel29). @@ -942,17 +857,20 @@ You will find the complete changelog for release 2.0.0, including bug fixes, imp - [1740](https://github.com/nginxinc/kubernetes-ingress/pull/1740) Publish docs using Hugo and Netlify. - [1702](https://github.com/nginxinc/kubernetes-ingress/pull/1702) Add security recommendations documentation. -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.11.0. - Add new parameters to the Chart: `controller.pod.extraLabels`. Added in [1884](https://github.com/nginxinc/kubernetes-ingress/pull/1884). -### Dependencies +CHANGES: - [1855](https://github.com/nginxinc/kubernetes-ingress/pull/1855) Update minimum Kubernetes version to 1.19; remove the `-use-ingress-class-only` command-line argument, which doesn't work with Kubernetes >= 1.19. +- [1721](https://github.com/nginxinc/kubernetes-ingress/pull/1721) Increase default reload timeout to 60s: the Ingress Controller will wait for 60s for NGINX to start or reload. Previously, the default was 4 seconds. +- [2009](https://github.com/nginxinc/kubernetes-ingress/pull/2009) Increase default upstream zone size for NGINX Plus. See the INCREASED UPSTREAM ZONES section below. - Update NGINX Plus version to R25. **Note**: images with NGINX App Protect will continue to use R24 until App Protect 3.6 is released. - Update NGINX version to 1.21.3. -### Upgrade +UPGRADE: - For NGINX, use the 2.0.0 image from our DockerHub. - For NGINX Plus, use the 2.0.0 from the F5 Container registry or build your own image using the 2.0.0 source code. @@ -984,34 +902,34 @@ Please also read the [release 1.10 changelog](https://docs.nginx.com/nginx-ingre Note that 2.1.0 will remove support for the v1alpha1 version of the Policy. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.22. -
- -## 1.12.1 +## NGINX Ingress Controller 1.12.1 8 September 2021 -### Dependencies +CHANGES: - Update NGINX App Protect version to 3.5. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.10.1. + +UPGRADE: - For NGINX, use the 1.12.1 image from our DockerHub: `nginx/nginx-ingress:1.12.1`, `nginx/nginx-ingress:1.12.1-alpine` or `nginx/nginx-ingress:1.12.1-ubi` -- For NGINX Plus, use the 1.12.1 image from the F5 Container Registry - see [the documentation here](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) +- For NGINX Plus, use the 1.12.1 image from the F5 Container Registry - see [the documentation here]({{< relref "installation/nic-images/pulling-ingress-controller-image" >}}) - Alternatively, you can also build your own image using the 1.12.1 source code. - For Helm, use version 0.10.1 of the chart. -
- -## 1.12.0 +## NGINX Ingress Controller 1.12.0 30 June 2021 -### Overview +OVERVIEW: Release 1.12.0 includes: @@ -1022,17 +940,23 @@ Release 1.12.0 includes: You will find the complete changelog for release 1.12.0, including bug fixes, improvements, and changes below. -### Features +FEATURES: - [1633](https://github.com/nginxinc/kubernetes-ingress/pull/1633) Support match in TransportServer health checks. - [1619](https://github.com/nginxinc/kubernetes-ingress/pull/1619) Add AWS Marketplace Entitlement verification. - [1480](https://github.com/nginxinc/kubernetes-ingress/pull/1480) Add max connections to TransportServer. - [1479](https://github.com/nginxinc/kubernetes-ingress/pull/1479) Add load balancing method to TransportServer. - [1466](https://github.com/nginxinc/kubernetes-ingress/pull/1466) Support snippets in TransportServer. + +FEATURES FOR NGINX APP PROTECT: + - [1578](https://github.com/nginxinc/kubernetes-ingress/pull/1578) Add support for CSRF protection in APPolicy. - [1513](https://github.com/nginxinc/kubernetes-ingress/pull/1513) Support multiple log security configs in Ingresses. - [1481](https://github.com/nginxinc/kubernetes-ingress/pull/1481) Add support for user defined browsers in APPolicy. - [1411](https://github.com/nginxinc/kubernetes-ingress/pull/1411) Add unary gRPC support in APPolicy. + +IMPROVEMENTS: + - [1671](https://github.com/nginxinc/kubernetes-ingress/pull/1671) Simplify Dockerfile stages for Debian. - [1652](https://github.com/nginxinc/kubernetes-ingress/pull/1652) Add HTTPS option to Prometheus endpoint. - [1646](https://github.com/nginxinc/kubernetes-ingress/pull/1646) Improve Dockerfile. @@ -1041,91 +965,98 @@ You will find the complete changelog for release 1.12.0, including bug fixes, im - [1500](https://github.com/nginxinc/kubernetes-ingress/pull/1500) Support ssl_reject_handshake in Ingress and VS. - [1494](https://github.com/nginxinc/kubernetes-ingress/pull/1494) Add logs around NGINX Plus binary/flag mismatch. - [1492](https://github.com/nginxinc/kubernetes-ingress/pull/1492) Update the IC so that GlobalConfiguration is not mandatory when configured. -- [1500](https://github.com/nginxinc/kubernetes-ingress/pull/1500) Support ssl_reject_handshake in Ingress and VS. Previously, to handle missing or invalid TLS Secrets in Ingress and VirtualServer resources, the Ingress Controller would configure NGINX to break any attempts for clients to establish TLS connections to the affected hosts using `ssl_ciphers NULL;` in the NGINX configuration. The method didn't work for TLS v1.3. Now the Ingress Controller uses `ssl_reject_handshake on;`, which works for TLS v1.3. - Documentation improvements: [1649](https://github.com/nginxinc/kubernetes-ingress/pull/1649). -### Fixes +FIXES: - [1658](https://github.com/nginxinc/kubernetes-ingress/pull/1658) Add missing njs module to the openshift-image-nap-plus image. - [1654](https://github.com/nginxinc/kubernetes-ingress/pull/1654) Fix incorrect configuration and unexpected warnings about Secrets at the IC start. - [1501](https://github.com/nginxinc/kubernetes-ingress/pull/1501) Fix ungraceful shutdown of NGINX. - Documentation fixes: [1668](https://github.com/nginxinc/kubernetes-ingress/pull/1668), [1594](https://github.com/nginxinc/kubernetes-ingress/pull/1594) thanks to [shaggy245](https://github.com/shaggy245), [1563](https://github.com/nginxinc/kubernetes-ingress/pull/1563), [1551](https://github.com/nginxinc/kubernetes-ingress/pull/1551). -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.10.0. - Add new parameters to the Chart: `prometheus.scheme`, `prometheus.secret`. Added in [1652](https://github.com/nginxinc/kubernetes-ingress/pull/1652). -### Dependencies +CHANGES: - [1604](https://github.com/nginxinc/kubernetes-ingress/pull/1604) Update NGINX Plus to R24. Previously, the Dockerfile had a fixed NGINX Plus version. Now the Dockerfile has a floating version that corresponds to the latest major NGINX Plus version. In the event of a patch version of NGINX Plus being released, make sure to rebuild your image to get the latest version (previously, we released a new Ingress Controller release in that case). Additionally, the AppProtect related packages are no longer fixed -- the Dockerfile will always install the latest version of the packages that work with the latest NGINX Plus version. +- [1500](https://github.com/nginxinc/kubernetes-ingress/pull/1500) Support ssl_reject_handshake in Ingress and VS. Previously, to handle missing or invalid TLS Secrets in Ingress and VirtualServer resources, the Ingress Controller would configure NGINX to break any attempts for clients to establish TLS connections to the affected hosts using `ssl_ciphers NULL;` in the NGINX configuration. The method didn't work for TLS v1.3. Now the Ingress Controller uses `ssl_reject_handshake on;`, which works for TLS v1.3. +- Update NGINX Plus version to R24. - Update NGINX version to 1.21.0. -### Upgrade +UPGRADE: - For NGINX, use the 1.12.0 image from our DockerHub: `nginx/nginx-ingress:1.12.0`, `nginx/nginx-ingress:1.12.0-alpine` or `nginx-ingress:1.12.0-ubi` - For NGINX Plus, please build your own image using the 1.12.0 source code. - For Helm, use version 0.10.0 of the chart. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.16-1.21. -
- -## 1.11.3 +## NGINX Ingress Controller 1.11.3 25 May 2021 -### Dependencies +CHANGES: - Update NGINX version to 1.21.0. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.9.3. + +UPGRADE: - For NGINX, use the 1.11.3 image from our DockerHub: `nginx/nginx-ingress:1.11.3`, `nginx/nginx-ingress:1.11.3-alpine` or `nginx/nginx-ingress:1.11.3-ubi` - For NGINX Plus, please build your own image using the 1.11.3 source code. - For Helm, use version 0.9.3 of the chart. -
- -## 1.11.2 +## NGINX Ingress Controller 1.11.2 19 May 2021 -### Dependencies +CHANGES: - Update NGINX Plus version to R23 P1. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.9.2. + +UPGRADE: - For NGINX, use the 1.11.2 image from our DockerHub: `nginx/nginx-ingress:1.11.2`, `nginx/nginx-ingress:1.11.2-alpine` or `nginx/nginx-ingress:1.11.2-ubi` - For NGINX Plus, please build your own image using the 1.11.2 source code. - For Helm, use version 0.9.2 of the chart. -
- -## 1.11.1 +## NGINX Ingress Controller 1.11.1 7 April 2021 -### Dependencies +CHANGES: - Update NGINX version to 1.19.9. - Update the OpenSSL libraries used in the UBI images +- Fixed documentation in the Helm README + +HELM CHART: -### Upgrade +- The version of the Helm chart is now 0.9.1. + +UPGRADE: - For NGINX, use the 1.11.1 image from our DockerHub: `nginx/nginx-ingress:1.11.1`, `nginx/nginx-ingress:1.11.1-alpine` or `nginx/nginx-ingress:1.11.1-ubi` - For NGINX Plus, please build your own image using the 1.11.1 source code. - For Helm, use version 0.9.1 of the chart. -
- -## 1.11.0 +## NGINX Ingress Controller 1.11.0 31 March 2021 -### Overview +OVERVIEW: Release 1.11.0 includes: @@ -1135,7 +1066,7 @@ Release 1.11.0 includes: You will find the complete changelog for release 1.11.0, including bug fixes, improvements, and changes below. -### Features +FEATURES: - [1317](https://github.com/nginxinc/kubernetes-ingress/pull/1317) Add status field to Policy resource. - [1449](https://github.com/nginxinc/kubernetes-ingress/pull/1449) Add support for ClusterIP in upstreams in VirtualServers/VirtualServerRoutes. @@ -1145,7 +1076,13 @@ You will find the complete changelog for release 1.11.0, including bug fixes, im - [1382](https://github.com/nginxinc/kubernetes-ingress/pull/1382) Add passive health checks to TransportServer. - [1346](https://github.com/nginxinc/kubernetes-ingress/pull/1346) Add configurable timeouts to TransportServer. - [1297](https://github.com/nginxinc/kubernetes-ingress/pull/1297) Support custom return in the default server. Thanks to [030](https://github.com/030). + +FEATURES FOR NGINX APP PROTECT: + - [1378](https://github.com/nginxinc/kubernetes-ingress/pull/1378) Add WAF Policy. + +IMPROVEMENTS: + - [1420](https://github.com/nginxinc/kubernetes-ingress/pull/1420) Support IngressClassName in TransportServer. - [1415](https://github.com/nginxinc/kubernetes-ingress/pull/1415) Handle host and listener collisions for TransportServer resource. - [1322](https://github.com/nginxinc/kubernetes-ingress/pull/1322) Improve VirtualServer/VirtualServerRoute warnings for Policies. @@ -1153,63 +1090,64 @@ You will find the complete changelog for release 1.11.0, including bug fixes, im - [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) Refactor Dockerfile and Makefile. - Documentation improvements: [1320](https://github.com/nginxinc/kubernetes-ingress/pull/1320), [1326](https://github.com/nginxinc/kubernetes-ingress/pull/1326), and [1377](https://github.com/nginxinc/kubernetes-ingress/pull/1377). -### Fixes +FIXES: - [1457](https://github.com/nginxinc/kubernetes-ingress/pull/1457) Wait for caches to sync when the Ingress Controller starts. - [1444](https://github.com/nginxinc/kubernetes-ingress/pull/1444) Fix setting host header in action proxy in VirtualServer/VirtualServerRoute. - [1396](https://github.com/nginxinc/kubernetes-ingress/pull/1396) Fix reload timeout calculation for verifying NGINX reloads. -### Dependencies +HELM CHART: + +- The version of the helm chart is now 0.9.0. + +CHANGES: - [1455](https://github.com/nginxinc/kubernetes-ingress/pull/1455) Update NGINX version to 1.19.8. - [1428](https://github.com/nginxinc/kubernetes-ingress/pull/1428) Update Nginx App Protect version to 3.0. **Note**: [The Advanced gRPC Protection for Unary Traffic](/nginx-app-protect/configuration/#advanced-grpc-protection-for-unary-traffic) is not currently supported. -### Known Issues +KNOWN ISSUES: - [1448](https://github.com/nginxinc/kubernetes-ingress/issues/1448) When an Ingress Controller pod starts, it can report warnings about missing secrets for Ingress and other resources that reference secrets. Those warnings are intermittent - once the Ingress Controller fully processes the resources of the cluster, it will clear the warnings. Only after that, the Ingress Controller will become ready to accept client traffic - its readiness probe will succeed. -### Upgrade +UPGRADE: - For NGINX, use the 1.11.0 image from our DockerHub: `nginx/nginx-ingress:1.11.0`, `nginx/nginx-ingress:1.11.0-alpine` or `nginx-ingress:1.11.0-ubi` - For NGINX Plus, please build your own image using the 1.11.0 source code. - For Helm, use version 0.9.0 of the chart. -- [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) improved the Makefile. As a result, the commands for building the Ingress Controller image were changed. See the updated commands [here](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/#building-the-image-and-pushing-it-to-the-private-registry). +- [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) improved the Makefile. As a result, the commands for building the Ingress Controller image were changed. See the updated commands [here]({{< relref "installation/building-nginx-ingress-controller#build-image-push-to-private-repo" >}}). - [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) also consolidated all Dockerfiles into a single Dockerfile. If you customized any of the Dockerfiles, make sure to port the changes to the new Dockerfile. - [1288](https://github.com/nginxinc/kubernetes-ingress/pull/1288) further improved validation of Ingress annotations. See this [document](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#validation) to learn more about which annotations are validated. Note that the Ingress Controller will reject resources with invalid annotations, which means clients will see `404` responses from NGINX. Before upgrading, ensure the Ingress resources don't have annotations with invalid values. Otherwise, after the upgrade, the Ingress Controller will reject such resources. - [1457](https://github.com/nginxinc/kubernetes-ingress/pull/1457) fixed the bug when an Ingress Controller pod could become ready before it generated the configuration for all relevant resources in the cluster. The fix also requires that the Ingress Controller can successfully list the relevant resources from the Kubernetes API. For example, if the `-enable-custom-resources` cli argument is `true` (which is the default), the VirtualServer, VirtualServerRoute, TransportServer, and Policy CRDs must be created in the cluster, so that the Ingress Controller can list them. This is similar to other custom resources -- see the list [here](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/#create-custom-resources). Thus, before upgrading, make sure that the CRDs are created in the cluster. Otherwise, the Ingress Controller pods will not become ready. -### Supported Platforms +SUPPORTED PLATFORMS: We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.16-1.20. -
- -## 1.10.1 +## NGINX Ingress Controller 1.10.1 16 March 2021 -### Dependencies +CHANGES: - Update NGINX version to 1.19.8. - Add Kubernetes 1.20 support. +- [1373](https://github.com/nginxinc/kubernetes-ingress/pull/1373), [1439](https://github.com/nginxinc/kubernetes-ingress/pull/1439), [1440](https://github.com/nginxinc/kubernetes-ingress/pull/1440): Fix various issues in the Makefile. In 1.10.0, a bug was introduced that prevented building Ingress Controller images on versions of make < 4.1. -### Fixes +HELM CHART: -- [1373](https://github.com/nginxinc/kubernetes-ingress/pull/1373), [1439](https://github.com/nginxinc/kubernetes-ingress/pull/1439), [1440](https://github.com/nginxinc/kubernetes-ingress/pull/1440): Fix various issues in the Makefile. In 1.10.0, a bug was introduced that prevented building Ingress Controller images on versions of make < 4.1. +- The version of the Helm chart is now 0.8.1. -### Upgrade +UPGRADE: - For NGINX, use the 1.10.1 image from our DockerHub: `nginx/nginx-ingress:1.10.1`, `nginx/nginx-ingress:1.10.1-alpine` or `nginx/nginx-ingress:1.10.1-ubi` - For NGINX Plus, please build your own image using the 1.10.1 source code. - For Helm, use version 0.8.1 of the chart. -
- -## 1.10.0 +## NGINX Ingress Controller 1.10.0 26 January 2021 -### Overview +OVERVIEW: Release 1.10.0 includes: @@ -1221,36 +1159,46 @@ Release 1.10.0 includes: You will find the complete changelog for release 1.10.0, including bug fixes, improvements, and changes below. -### Features +FEATURES FOR POLICY RESOURCE: - [1304](https://github.com/nginxinc/kubernetes-ingress/pull/1304) Add Open ID Connect policy. + +FEATURES FOR NGINX APP PROTECT: + - [1281](https://github.com/nginxinc/kubernetes-ingress/pull/1281) Add support for App Protect User Defined Signatures. + +FEATURES: + - [1266](https://github.com/nginxinc/kubernetes-ingress/pull/1266) Add workqueue metrics to Prometheus metrics. - [1233](https://github.com/nginxinc/kubernetes-ingress/pull/1233) Annotate tcp metrics with k8s object labels. - [1231](https://github.com/nginxinc/kubernetes-ingress/pull/1231) Support k8s objects variables in log format. + +IMPROVEMENTS: + - [1270](https://github.com/nginxinc/kubernetes-ingress/pull/1270) and [1277](https://github.com/nginxinc/kubernetes-ingress/pull/1277) Improve validation of Ingress annotations. - [1265](https://github.com/nginxinc/kubernetes-ingress/pull/1265) Report warnings for misconfigured TLS and JWK secrets. - [1262](https://github.com/nginxinc/kubernetes-ingress/pull/1262) Use setcap(8) only once. [1263](https://github.com/nginxinc/kubernetes-ingress/pull/1263) Use chown(8) only once. [1264](https://github.com/nginxinc/kubernetes-ingress/pull/1264) Use mkdir(1) only once. Thanks to [Sergey A. Osokin](https://github.com/osokin). - [1256](https://github.com/nginxinc/kubernetes-ingress/pull/1256) and [1260](https://github.com/nginxinc/kubernetes-ingress/pull/1260) Improve handling of secret resources. - [1240](https://github.com/nginxinc/kubernetes-ingress/pull/1240) Validate TLS and CA secrets. - [1235](https://github.com/nginxinc/kubernetes-ingress/pull/1235) Use buildkit secret flag for NGINX plus images. -- [1290](https://github.com/nginxinc/kubernetes-ingress/pull/1290) Graduate policy resource and accessControl policy to generally available. -- [1225](https://github.com/nginxinc/kubernetes-ingress/pull/1225) Require secrets to have types. -- [1237](https://github.com/nginxinc/kubernetes-ingress/pull/1237) Deprecate support for helm2 clients. - Documentation improvements: [1282](https://github.com/nginxinc/kubernetes-ingress/pull/1282), [1293](https://github.com/nginxinc/kubernetes-ingress/pull/1293), [1303](https://github.com/nginxinc/kubernetes-ingress/pull/1303), [1315](https://github.com/nginxinc/kubernetes-ingress/pull/1315). -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.8.0. - [1290](https://github.com/nginxinc/kubernetes-ingress/pull/1290) Add new preview policies parameter to chart. `controller.enablePreviewPolicies` was added. - [1232](https://github.com/nginxinc/kubernetes-ingress/pull/1232) Replace deprecated imagePullSecrets helm setting. `controller.serviceAccount.imagePullSecrets` was removed. `controller.serviceAccount.imagePullSecretName` was added. - [1228](https://github.com/nginxinc/kubernetes-ingress/pull/1228) Fix installation of ingressclass on Kubernetes versions `v1.18.x-*` -### Dependencies +CHANGES: - [1299](https://github.com/nginxinc/kubernetes-ingress/pull/1299) Update NGINX App Protect version to 2.3 and debian distribution to `debian:buster-slim`. - [1291](https://github.com/nginxinc/kubernetes-ingress/pull/1291) Update NGINX OSS to `1.19.6`. Update NGINX Plus to `R23`. +- [1290](https://github.com/nginxinc/kubernetes-ingress/pull/1290) Graduate policy resource and accessControl policy to generally available. +- [1225](https://github.com/nginxinc/kubernetes-ingress/pull/1225) Require secrets to have types. +- [1237](https://github.com/nginxinc/kubernetes-ingress/pull/1237) Deprecate support for helm2 clients. -### Upgrade +UPGRADE: - For NGINX, use the 1.10.0 image from our DockerHub: `nginx/nginx-ingress:1.10.0`, `nginx/nginx-ingress:1.10.0-alpine` or `nginx-ingress:1.10.0-ubi` - For NGINX Plus, please build your own image using the 1.10.0 source code. @@ -1301,35 +1249,32 @@ NOTES: - Helm 2 clients are no longer supported due to reaching End of Life: -
- -## 1.9.1 +## NGINX Ingress Controller 1.9.1 23 November 2020 -### Dependencies - -- Update the base ubi images to 8.3. - -### Features +CHANGES: - Fix deployment of ingressclass resource via helm on some versions of Kubernetes. +- Update the base ubi images to 8.3. - Renew CA cert for egress-mtls example. - Add imagePullSecretName support to helm chart. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.7.1. + +UPGRADE: - For NGINX, use the 1.9.1 image from our DockerHub: `nginx/nginx-ingress:1.9.1`, `nginx/nginx-ingress:1.9.1-alpine` or `nginx/nginx-ingress:1.9.1-ubi` - For NGINX Plus, please build your own image using the 1.9.1 source code. - For Helm, use version 0.7.1 of the chart. -
- -## 1.9.0 +## NGINX Ingress Controller 1.9.0 20 October 2020 -### Overview +OVERVIEW: Release 1.9.0 includes: @@ -1340,21 +1285,30 @@ Release 1.9.0 includes: You will find the complete changelog for release 1.9.0, including bug fixes, improvements, and changes below. -### Features +FEATURES FOR POLICY RESOURCE: - [1180](https://github.com/nginxinc/kubernetes-ingress/pull/1180) Add support for EgressMTLS. - [1166](https://github.com/nginxinc/kubernetes-ingress/pull/1166) Add IngressMTLS policy support. - [1154](https://github.com/nginxinc/kubernetes-ingress/pull/1154) Add JWT policy support. - [1120](https://github.com/nginxinc/kubernetes-ingress/pull/1120) Add RateLimit policy support. - [1058](https://github.com/nginxinc/kubernetes-ingress/pull/1058) Support policies in VS routes and VSR subroutes. + +FEATURES FOR NGINX APP PROTECT: + - [1147](https://github.com/nginxinc/kubernetes-ingress/pull/1147) Add option to specify other log destinations in AppProtect. - [1131](https://github.com/nginxinc/kubernetes-ingress/pull/1131) Update packages and CRDs to AppProtect 2.0. This update includes features such as: [JSON Schema Validation](https://docs.nginx.com/nginx-app-protect/configuration#applying-a-json-schema), [User-Defined URLs](https://docs.nginx.com/nginx-app-protect/configuration/#user-defined-urls) and [User-Defined Parameters](https://docs.nginx.com/nginx-app-protect/configuration/#user-defined-parameters). See the [release notes](https://docs.nginx.com/nginx-app-protect/releases/#release-2-0) for a complete feature list. - [1100](https://github.com/nginxinc/kubernetes-ingress/pull/1100) Add external references to AppProtect. - [1085](https://github.com/nginxinc/kubernetes-ingress/pull/1085) Add installation of threat campaigns package. + +FEATURES: + - [1133](https://github.com/nginxinc/kubernetes-ingress/pull/1133) Add support for IngressClass resources. - [1130](https://github.com/nginxinc/kubernetes-ingress/pull/1130) Add prometheus latency collector. - [1076](https://github.com/nginxinc/kubernetes-ingress/pull/1076) Add prometheus worker process metrics. - [1075](https://github.com/nginxinc/kubernetes-ingress/pull/1075) Add support for NGINX Service Mesh internal routes. + +IMPROVEMENTS: + - [1178](https://github.com/nginxinc/kubernetes-ingress/pull/1178) Resolve host collisions in VirtualServer and Ingresses. - [1158](https://github.com/nginxinc/kubernetes-ingress/pull/1158) Support variables in action proxy headers. - [1137](https://github.com/nginxinc/kubernetes-ingress/pull/1137) Add pod_owner label to metrics when -spire-agent-address is set. @@ -1365,22 +1319,23 @@ You will find the complete changelog for release 1.9.0, including bug fixes, imp - [1078](https://github.com/nginxinc/kubernetes-ingress/pull/1078) Remove trailing blank lines in vs/vsr snippets. - Documentation improvements: [1083](https://github.com/nginxinc/kubernetes-ingress/pull/1083), [1092](https://github.com/nginxinc/kubernetes-ingress/pull/1092), [1089](https://github.com/nginxinc/kubernetes-ingress/pull/1089), [1174](https://github.com/nginxinc/kubernetes-ingress/pull/1174), [1175](https://github.com/nginxinc/kubernetes-ingress/pull/1175), [1171](https://github.com/nginxinc/kubernetes-ingress/pull/1171). -### Fixes +BUGFIXES: - [1179](https://github.com/nginxinc/kubernetes-ingress/pull/1179) Fix TransportServers in debian AppProtect image. - [1129](https://github.com/nginxinc/kubernetes-ingress/pull/1129) Support real-ip in default server. - [1110](https://github.com/nginxinc/kubernetes-ingress/pull/1110) Add missing threat campaigns key to AppProtect CRD. -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.7.0 - [1105](https://github.com/nginxinc/kubernetes-ingress/pull/1105) Fix GlobalConfiguration support in helm chart. - Add new parameters to the Chart: `controller.setAsDefaultIngress`, `controller.enableLatencyMetrics`. Added in [1133](https://github.com/nginxinc/kubernetes-ingress/pull/1133) and [1148](https://github.com/nginxinc/kubernetes-ingress/pull/1148). -### Dependencies +CHANGES: - [1182](https://github.com/nginxinc/kubernetes-ingress/pull/1182) Update NGINX version to 1.19.3. -### Upgrade +UPGRADE: - For NGINX, use the 1.9.0 image from our DockerHub: `nginx/nginx-ingress:1.9.0`, `nginx/nginx-ingress:1.9.0-alpine` or `nginx-ingress:1.9.0-ubi` - For NGINX Plus, please build your own image using the 1.9.0 source code. @@ -1388,39 +1343,39 @@ You will find the complete changelog for release 1.9.0, including bug fixes, imp For Kubernetes >= 1.18, when upgrading using the [manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/), make sure to update the [ClusterRole](https://github.com/nginxinc/kubernetes-ingress/blob/v1.9.0/deployments/rbac/rbac.yaml) and create the [IngressClass resource](https://github.com/nginxinc/kubernetes-ingress/blob/v1.9.0/deployments/common/ingress-class.yaml), which is required for Kubernetes >= 1.18. Otherwise, the Ingress Controller will fail to start. If you run multiple NGINX Ingress Controllers in the cluster, each Ingress Controller must have its own IngressClass resource. As the `-use-ingress-class-only` argument is now ignored (see NOTES), make sure your Ingress resources have the `ingressClassName` field or the `kubernetes.io/ingress.class` annotation set to the name of the IngressClass resource. Otherwise, the Ingress Controller will ignore them. -#### Helm Upgrade +HELM UPGRADE: - If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs will not be upgraded. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. - For Kubernetes >= 1.18, a dedicated IngressClass resource, which is configured by `controller.ingressClass`, is required per helm release. Ensure `controller.ingressClass` is not set to the name of the IngressClass of other releases or Ingress Controllers. As the `controller.useIngressClassOnly` parameter is now ignored (see NOTES), make sure your Ingress resources have the `ingressClassName` field or the `kubernetes.io/ingress.class` annotation set to the value of `controller.ingressClass`. Otherwise, the Ingress Controller will ignore them. -### Notes +NOTES: - When using Kubernetes >= 1.18, the `-use-ingress-class-only` command-line argument is now ignored, and the Ingress Controller will only process resources that belong to its class. See [IngressClass doc](https://docs.nginx.com/nginx-ingress-controller/installation/running-multiple-ingress-controllers/#ingress-class) for more details. - For Kubernetes >= 1.18, a dedicated IngressClass resource, which is configured by `controller.ingressClass`, is required per helm release. When upgrading or installing releases, ensure `controller.ingressClass` is not set to the name of the IngressClass of other releases or Ingress Controllers. -
- -## 1.8.1 +## NGINX Ingress Controller 1.8.1 14 August 2020 -### Dependencies +CHANGES: - Update NGINX version to 1.19.2. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.6.1. + +UPGRADE: - For NGINX, use the 1.8.1 image from our DockerHub: `nginx/nginx-ingress:1.8.1`, `nginx/nginx-ingress:1.8.1-alpine` or `nginx/nginx-ingress:1.8.1-ubi` - For NGINX Plus, please build your own image using the 1.8.1 source code. - For Helm, use version 0.6.1 of the chart. -
- -## 1.8.0 +## NGINX Ingress Controller 1.8.0 22 July 2020 -### Overview +OVERVIEW: Release 1.8.0 includes: @@ -1440,83 +1395,89 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [994](https://github.com/nginxinc/kubernetes-ingress/pull/994): Support Class Field in VS/VSR. - [973](https://github.com/nginxinc/kubernetes-ingress/pull/973): Add status to VirtualServer and VirtualServerRoute. -### Features +FEATURES: - [1035](https://github.com/nginxinc/kubernetes-ingress/pull/1035): Support for App Protect module. - [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029): Add readiness endpoint. + +IMPROVEMENTS: + - [995](https://github.com/nginxinc/kubernetes-ingress/pull/995): Emit event for orphaned VirtualServerRoutes. -- [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029): Add readiness endpoint. The Ingress Controller now exposes a readiness endpoint on port `8081` and the path `/nginx-ready`. The endpoint returns a `200` response after the Ingress Controller finishes the initial configuration of NGINX at the start. The pod template was updated to use that endpoint in a readiness probe. -- [980](https://github.com/nginxinc/kubernetes-ingress/pull/980): Enable leader election by default. - Documentation improvements: [946](https://github.com/nginxinc/kubernetes-ingress/pull/946) thanks to [谭九鼎](https://github.com/imba-tjd), [948](https://github.com/nginxinc/kubernetes-ingress/pull/948), [972](https://github.com/nginxinc/kubernetes-ingress/pull/972), [965](https://github.com/nginxinc/kubernetes-ingress/pull/965). -### Fixes +BUGFIXES: - [1030](https://github.com/nginxinc/kubernetes-ingress/pull/1030): Fix port range validation in cli arguments. - [953](https://github.com/nginxinc/kubernetes-ingress/pull/953): Fix error logging of master/minion ingresses. -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.6.0. - Add new parameters to the Chart: `controller.appprotect.enable`, `controller.globalConfiguration.create`, `controller.globalConfiguration.spec`, `controller.readyStatus.enable`, `controller.readyStatus.port`, `controller.config.annotations`, `controller.reportIngressStatus.annotations`. Added in [1035](https://github.com/nginxinc/kubernetes-ingress/pull/1035), [1034](https://github.com/nginxinc/kubernetes-ingress/pull/1034), [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029), [1003](https://github.com/nginxinc/kubernetes-ingress/pull/1003) thanks to [RubyLangdon](https://github.com/RubyLangdon). - [1047](https://github.com/nginxinc/kubernetes-ingress/pull/1047) and [1009](https://github.com/nginxinc/kubernetes-ingress/pull/1009): Change how Helm manages the custom resource definitions (CRDs) to support installing multiple Ingress Controller releases. **Note**: If you're using the custom resources (`controller.enableCustomResources` is set to `true`), this is a breaking change. See the HELM UPGRADE section below for the upgrade instructions. -### Dependencies +CHANGES: - Update NGINX version to 1.19.1. - Update NGINX Plus to R22. +- [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029): Add readiness endpoint. The Ingress Controller now exposes a readiness endpoint on port `8081` and the path `/nginx-ready`. The endpoint returns a `200` response after the Ingress Controller finishes the initial configuration of NGINX at the start. The pod template was updated to use that endpoint in a readiness probe. +- [980](https://github.com/nginxinc/kubernetes-ingress/pull/980): Enable leader election by default. -### Upgrade +UPGRADE: - For NGINX, use the 1.8.0 image from our DockerHub: `nginx/nginx-ingress:1.8.0`, `nginx/nginx-ingress:1.8.0-alpine` or `nginx-ingress:1.8.0-ubi` - For NGINX Plus, please build your own image using the 1.8.0 source code. - For Helm, use version 0.6.0 of the chart. -#### Helm Upgrade +HELM UPGRADE: If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs and the corresponding custom resources will be removed from the cluster. Before upgrading, make sure to back up the custom resources. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to re-install the CRDs and then restore the custom resources. -### Notes +NOTES: - As part of installing a release, Helm will install the CRDs unless that step is disabled (see the [corresponding doc](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-helm/)). The installed CRDs include the CRDs for all Ingress Controller features, including the ones disabled by default (like App Protect with `aplogconfs.appprotect.f5.com` and `appolicies.appprotect.f5.com` CRDs). -
- -## 1.7.2 +## NGINX Ingress Controller 1.7.2 23 June 2020 -### Dependencies +CHANGES: - Update NGINX Plus version to R22. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.5.2. + +UPGRADE: - For NGINX, use the 1.7.2 image from our DockerHub: `nginx/nginx-ingress:1.7.2`, `nginx/nginx-ingress:1.7.2-alpine` or `nginx/nginx-ingress:1.7.2-ubi` - For NGINX Plus, please build your own image using the 1.7.2 source code. - For Helm, use version 0.5.2 of the chart. -
- -## 1.7.1 +## NGINX Ingress Controller 1.7.1 4 June 2020 -### Dependencies +CHANGES: - Update NGINX version to 1.19.0. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.5.1. + +UPGRADE: - For NGINX, use the 1.7.1 image from our DockerHub: `nginx/nginx-ingress:1.7.1`, `nginx/nginx-ingress:1.7.1-alpine` or `nginx/nginx-ingress:1.7.1-ubi` - For NGINX Plus, please build your own image using the 1.7.1 source code. - For Helm, use version 0.5.1 of the chart. -
- -## 1.7.0 +## NGINX Ingress Controller 1.7.0 30 April 2020 -### Overview +OVERVIEW: Release 1.7.0 includes: @@ -1534,7 +1495,7 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [868](https://github.com/nginxinc/kubernetes-ingress/pull/868): Add OpenAPI CRD schema validation. - [847](https://github.com/nginxinc/kubernetes-ingress/pull/847): Add support for error pages for VS/VSR. -### Features +FEATURES: - [902](https://github.com/nginxinc/kubernetes-ingress/pull/902): Add TransportServer and GlobalConfiguration Resources. - [894](https://github.com/nginxinc/kubernetes-ingress/pull/894): Add Dockerfile for NGINX Open Source for Openshift. @@ -1542,29 +1503,33 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [852](https://github.com/nginxinc/kubernetes-ingress/pull/852): Add default-server-access-log-off to configmap. - [845](https://github.com/nginxinc/kubernetes-ingress/pull/845): Add log-format-escaping and stream-log-format-escaping configmap keys. Thanks to [Alexey Maslov](https://github.com/alxmsl). - [827](https://github.com/nginxinc/kubernetes-ingress/pull/827): Add ingress class label to all Prometheus metrics. + +IMPROVEMENTS: + - [850](https://github.com/nginxinc/kubernetes-ingress/pull/850): Extend redirect URI validation with protocol check in VS/VSR. - [832](https://github.com/nginxinc/kubernetes-ingress/pull/832): Update the examples to run the `nginxdemos/nginx-hello:plain-text` image, that doesn't require root user. - [825](https://github.com/nginxinc/kubernetes-ingress/pull/825): Add multi-stage docker builds. -- [852](https://github.com/nginxinc/kubernetes-ingress/pull/852): Add default-server-access-log-off to configmap. The access logs for the default server are now enabled by default. -- [847](https://github.com/nginxinc/kubernetes-ingress/pull/847): Add support for error pages for VS/VSR. The PR affects how the Ingress Controller generates configuration for VirtualServer and VirtualServerRoutes. See [this comment](https://github.com/nginxinc/kubernetes-ingress/pull/847) for more details. -- [827](https://github.com/nginxinc/kubernetes-ingress/pull/827): Add ingress class label to all Prometheus metrics. Every Prometheus metric exposed by the Ingress Controller now includes the label `class` with the value of the Ingress Controller class (by default `nginx`), -- [825](https://github.com/nginxinc/kubernetes-ingress/pull/825): Add multi-stage docker builds. When building the Ingress Controller image in Docker, we now use a multi-stage docker build. -### Fixes +BUGFIXES: - [828](https://github.com/nginxinc/kubernetes-ingress/pull/828): Fix error messages for actions of the type return. -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.5.0. - Add new parameters to the Chart: `controller.enableTLSPassthrough`, `controller.volumes`, `controller.volumeMounts`, `controller.priorityClassName`. Added in [921](https://github.com/nginxinc/kubernetes-ingress/pull/921), [878](https://github.com/nginxinc/kubernetes-ingress/pull/878), [807](https://github.com/nginxinc/kubernetes-ingress/pull/807) thanks to [Greg Snow](https://github.com/gsnegovskiy). -### Dependencies +CHANGES: - Update NGINX version to 1.17.10. - Update NGINX Plus to R21. - [854](https://github.com/nginxinc/kubernetes-ingress/pull/854): Update the Debian base images for NGINX Plus to `debian:buster-slim`. +- [852](https://github.com/nginxinc/kubernetes-ingress/pull/852): Add default-server-access-log-off to configmap. The access logs for the default server are now enabled by default. +- [847](https://github.com/nginxinc/kubernetes-ingress/pull/847): Add support for error pages for VS/VSR. The PR affects how the Ingress Controller generates configuration for VirtualServer and VirtualServerRoutes. See [this comment](https://github.com/nginxinc/kubernetes-ingress/pull/847) for more details. +- [827](https://github.com/nginxinc/kubernetes-ingress/pull/827): Add ingress class label to all Prometheus metrics. Every Prometheus metric exposed by the Ingress Controller now includes the label `class` with the value of the Ingress Controller class (by default `nginx`), +- [825](https://github.com/nginxinc/kubernetes-ingress/pull/825): Add multi-stage docker builds. When building the Ingress Controller image in Docker, we now use a multi-stage docker build. -### Upgrade +UPGRADE: - For NGINX, use the 1.7.0 image from our DockerHub: `nginx/nginx-ingress:1.7.0`, `nginx/nginx-ingress:1.7.0-alpine` or `nginx-ingress:1.7.0-ubi` - For NGINX Plus, please build your own image using the 1.7.0 source code. @@ -1572,61 +1537,65 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: When upgrading using the [manifests](/nginx-ingress-controller/installation/installation-with-manifests/), make sure to deploy the new TransportServer CRD (`common/ts-definition.yaml`), as it is required by the Ingress Controller. Otherwise, you will get error messages in the Ingress Controller logs. -
- -## 1.6.3 +## NGINX Ingress Controller 1.6.3 6 March 2020 -### Dependencies +CHANGES: - Update NGINX version to 1.17.9. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.4.3. + +UPGRADE: - For NGINX, use the 1.6.3 image from our DockerHub: `nginx/nginx-ingress:1.6.3` or `nginx/nginx-ingress:1.6.3-alpine` - For NGINX Plus, please build your own image using the 1.6.3 source code. - For Helm, use version 0.4.3 of the chart. -
- -## 1.6.2 +## NGINX Ingress Controller 1.6.2 6 February 2020 -### Dependencies +CHANGES: - Update NGINX version to 1.17.8. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.4.2. + +UPGRADE: - For NGINX, use the 1.6.2 image from our DockerHub: `nginx/nginx-ingress:1.6.2` or `nginx/nginx-ingress:1.6.2-alpine` - For NGINX Plus, please build your own image using the 1.6.2 source code. - For Helm, use version 0.4.2 of the chart. -
- -## 1.6.1 +## NGINX Ingress Controller 1.6.1 14 January 2020 -### Dependencies +CHANGES: - Update NGINX version to 1.17.7. -### Upgrade +HELM CHART: + +- The version of the Helm chart is now 0.4.1. + +UPGRADE: - For NGINX, use the 1.6.1 image from our DockerHub: `nginx/nginx-ingress:1.6.1` or `nginx/nginx-ingress:1.6.1-alpine` - For NGINX Plus, please build your own image using the 1.6.1 source code. - For Helm, use version 0.4.1 of the chart. -
- -## 1.6.0 +## NGINX Ingress Controller 1.6.0 19 December 2019 -### Overview +OVERVIEW: Release 1.6.0 includes: @@ -1666,7 +1635,7 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [607](https://github.com/nginxinc/kubernetes-ingress/pull/607): Add fail-timeout and max-fails support to vs/vsr. - [596](https://github.com/nginxinc/kubernetes-ingress/pull/596): Add lb-method support in vs and vsr. -### Features +FEATURES: - [750](https://github.com/nginxinc/kubernetes-ingress/pull/750): Add support for health status uri customisation. - [691](https://github.com/nginxinc/kubernetes-ingress/pull/691): Helper Functions for custom annotations. @@ -1675,17 +1644,14 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [616](https://github.com/nginxinc/kubernetes-ingress/pull/616): Add proxy-send-timeout to configmap key and annotation. - [615](https://github.com/nginxinc/kubernetes-ingress/pull/615): Add support for Opentracing. - [614](https://github.com/nginxinc/kubernetes-ingress/pull/614): Add max-conns annotation. Thanks to [Victor Regalado](https://github.com/vrrs). + +IMPROVEMENTS: + - [678](https://github.com/nginxinc/kubernetes-ingress/pull/678): Increase defaults for server-names-hash-max-size and servers-names-hash-bucket-size ConfigMap keys. - [694](https://github.com/nginxinc/kubernetes-ingress/pull/694): Reject VS/VSR resources with enabled plus features for OSS. -- [799](https://github.com/nginxinc/kubernetes-ingress/pull/779): Enable CRDs by default. VirtualServer and VirtualServerRoute resources are now enabled by default. -- [772](https://github.com/nginxinc/kubernetes-ingress/pull/772): Update VS/VSR version from v1alpha1 to v1. Make sure to update the `apiVersion` of your VirtualServer and VirtualServerRoute resources. -- [748](https://github.com/nginxinc/kubernetes-ingress/pull/748): Add TLS redirect support in VirtualServer. The `redirect-to-https` and `ssl-redirect` ConfigMap keys no longer have any effect on generated configs for VirtualServer resources. -- [745](https://github.com/nginxinc/kubernetes-ingress/pull/745): Improve routing rules. Update the spec of VirtualServer and VirtualServerRoute accordingly. See YAML examples of the changes [here](https://github.com/nginxinc/kubernetes-ingress/pull/745). -- [710](https://github.com/nginxinc/kubernetes-ingress/pull/710): Run IC as non-root. Make sure to use the updated manifests to install/upgrade the Ingress Controller. -- [603](https://github.com/nginxinc/kubernetes-ingress/pull/603): Update apiVersion in Deployments and DaemonSets to apps/v1. - Documentation improvements: [713](https://github.com/nginxinc/kubernetes-ingress/pull/713) thanks to [Matthew Wahner](https://github.com/mattwahner). -### Fixes +BUGFIXES: - [788](https://github.com/nginxinc/kubernetes-ingress/pull/788): Fix VSR updates when namespace is set implicitly. - [736](https://github.com/nginxinc/kubernetes-ingress/pull/736): Init Ingress labeled metrics on start. @@ -1693,24 +1659,31 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [664](https://github.com/nginxinc/kubernetes-ingress/pull/664): Fix reporting events for Ingress minions. - [632](https://github.com/nginxinc/kubernetes-ingress/pull/632): Fix hsts support when not using SSL. Thanks to [Martín Fernández](https://github.com/bilby91). -### Helm Chart +HELM CHART: +- The version of the helm chart is now 0.4.0. - Add new parameters to the Chart: `controller.healthCheckURI`, `controller.resources`, `controller.logLevel`, `controller.customPorts`, `controller.service.customPorts`. Added in [750](https://github.com/nginxinc/kubernetes-ingress/pull/750), [636](https://github.com/nginxinc/kubernetes-ingress/pull/636) thanks to [Guilherme Oki](https://github.com/guilhermeoki), [600](https://github.com/nginxinc/kubernetes-ingress/pull/600), [581](https://github.com/nginxinc/kubernetes-ingress/pull/581) thanks to [Alex Meijer](https://github.com/ameijer-corsha). - [722](https://github.com/nginxinc/kubernetes-ingress/pull/722): Fix trailing leader election cm when using helm. This change might lead to a failed upgrade. See the helm upgrade instruction below. - [573](https://github.com/nginxinc/kubernetes-ingress/pull/573): Use Controller name value for app selectors. -### Dependencies +CHANGES: - Update NGINX versions to 1.17.6. - Update NGINX Plus version to R20. +- [799](https://github.com/nginxinc/kubernetes-ingress/pull/779): Enable CRDs by default. VirtualServer and VirtualServerRoute resources are now enabled by default. +- [772](https://github.com/nginxinc/kubernetes-ingress/pull/772): Update VS/VSR version from v1alpha1 to v1. Make sure to update the `apiVersion` of your VirtualServer and VirtualServerRoute resources. +- [748](https://github.com/nginxinc/kubernetes-ingress/pull/748): Add TLS redirect support in VirtualServer. The `redirect-to-https` and `ssl-redirect` ConfigMap keys no longer have any effect on generated configs for VirtualServer resources. +- [745](https://github.com/nginxinc/kubernetes-ingress/pull/745): Improve routing rules. Update the spec of VirtualServer and VirtualServerRoute accordingly. See YAML examples of the changes [here](https://github.com/nginxinc/kubernetes-ingress/pull/745). +- [710](https://github.com/nginxinc/kubernetes-ingress/pull/710): Run IC as non-root. Make sure to use the updated manifests to install/upgrade the Ingress Controller. +- [603](https://github.com/nginxinc/kubernetes-ingress/pull/603): Update apiVersion in Deployments and DaemonSets to apps/v1. -### Upgrade +UPGRADE: - For NGINX, use the 1.6.0 image from our DockerHub: `nginx/nginx-ingress:1.6.0` or `nginx/nginx-ingress:1.6.0-alpine` - For NGINX Plus, please build your own image using the 1.6.0 source code. - For Helm, use version 0.4.0 of the chart. -#### Helm upgrade +HELM UPGRADE: If leader election (the `controller.reportIngressStatus.enableLeaderElection` parameter) is enabled, when upgrading to the new version of the Helm chart: @@ -1719,8 +1692,6 @@ If leader election (the `controller.reportIngressStatus.enableLeaderElection` pa Otherwise, the helm upgrade will not succeed. -
- ## Previous Releases -To see the previous releases, see the [Releases page](https://github.com/nginxinc/kubernetes-ingress/releases) on the Ingress Controller GitHub repo. +To see the previous releases, see the [Releases page]({{< relref "releases.md" >}}) on the Ingress Controller GitHub repo. diff --git a/docs/content/technical-specifications.md b/docs/content/technical-specifications.md index ced188a7423..b1ad5669c25 100644 --- a/docs/content/technical-specifications.md +++ b/docs/content/technical-specifications.md @@ -1,7 +1,7 @@ --- title: Technical Specifications description: "NGINX Ingress Controller Technical Specifications." -weight: 2200 +weight: 200 doctypes: ["concept"] toc: true docs: "DOCS-617" @@ -10,20 +10,19 @@ docs: "DOCS-617" ## Supported NGINX Ingress Controller Versions -We advise users to run the most recent release of the NGINX Ingress Controller, and we issue software updates to the most recent release. We provide technical support for F5 customers who are using the most recent version of the NGINX Ingress Controller, and any version released within two years of the current release. +We recommend upgrading to the latest release of the NGINX Ingress Controller. We provide software updates for the most recent release. We provide technical support for F5 customers who are using the most recent version of NGINX Ingress Controller, and any version released within two years of the current release. -The 3.0.0 release supports `discovery.k8s.io/v1` API version of EndpointSlice, available from Kubernetes 1.21 onwards. -The 2.4.2 release is compatible with the Kubernetes Ingress v1 API. Therefore Kubernetes 1.19 and later. -The 1.12 release supports the Ingress v1beta1 API and continues to receive security fixes to support those unable to upgrade to Kubernetes 1.19 or later. The v1beta1 Ingress API was deprecated with Kubernetes release 1.19 and removed with the Kubernetes 1.22 release. +Release 3.0.0 provides support for the `discovery.k8s.io/v1` API version of EndpointSlice, available from Kubernetes 1.21 onwards. +Release 2.4.2 is compatible with the Kubernetes Ingress v1 API, available in Kubernetes 1.19 and later. +Release 1.12 supports the Ingress v1beta1 API and continues to receive security fixes to support environments running Kubernetes versions older than 1.19. The v1beta1 Ingress API was deprecated in Kubernetes release 1.19, and removed in the Kubernetes 1.22. ## Supported Kubernetes Versions -We explicitly test the NGINX Ingress Controller (NIC) on a range of Kubernetes platforms at each release, and the [release notes](/nginx-ingress-controller/releases) list which platforms were tested. We will provide technical support for the NGINX Ingress Controller (NIC) on any Kubernetes platform that is currently supported by its provider and which passes the [Kubernetes conformance tests](https://www.cncf.io/certification/software-conformance/). +We explicitly test NGINX Ingress Controller on a range of Kubernetes platforms for each release, and we list them in the [release notes]({{< relref "/releases.md" >}}). We provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider, and which passes the [Kubernetes conformance tests](https://www.cncf.io/certification/software-conformance/). -{{% table %}} +{{< bootstrap-table "table table-bordered table-striped table-responsive" >}} | NIC Version | Supported Kubernetes Version | NIC Helm Chart Version | NIC Operator Version | NGINX / NGINX Plus version | | --- | --- | --- | --- | --- | -| 3.3.2 | 1.28 - 1.22 | 1.0.2 | 2.0.2 | 1.25.3 / R30 | | 3.2.1 | 1.27 - 1.22 | 0.18.1 | 1.5.1 | 1.25.2 / R30 | | 3.1.1 | 1.26 - 1.22 | 0.17.1 | 1.4.2 | 1.23.4 / R29 | | 3.0.2 | 1.26 - 1.21 | 0.16.2 | 1.3.1 | 1.23.3 / R28 | @@ -39,64 +38,69 @@ We explicitly test the NGINX Ingress Controller (NIC) on a range of Kubernetes p | 1.8.1 | | 0.6.0 | 0.0.6 | 1.19.2 / R22 | | 1.7.2 | | 0.5.1 | 0.0.4 | 1.19.0 / R22 | | 1.6.3 | | 0.4.3 | -- | 1.17.9 / R21 | -{{% /table %}} +{{% /bootstrap-table %}} ## Supported Docker Images -We provide the following Docker images, which include NGINX/NGINX Plus bundled with the Ingress Controller binary. +We provide the following Docker images, which include NGINX or NGINX Plus bundled with the Ingress Controller binary. ### Images with NGINX -All images include NGINX 1.25.3. +_All images include NGINX 1.25.2._ -{{% table %}} -|Name | Base image | Third-party modules | DockerHub image | Architectures | -| ---| ---| ---| --- | --- | -|Alpine-based image | ``nginx:1.25.3-alpine``, which is based on ``alpine:3.18`` | NGINX OpenTracing module, OpenTracing library, OpenTracing tracers for Jaeger, Zipkin and Datadog | ``nginx/nginx-ingress:3.3.2-alpine`` | arm/v7, arm64, amd64, ppc64le, s390x | -|Debian-based image | ``nginx:1.25.3``, which is based on ``debian:12-slim`` | NGINX OpenTracing module, OpenTracing library, OpenTracing tracers for Jaeger, Zipkin and Datadog | ``nginx/nginx-ingress:3.3.2`` | arm/v7, arm64, amd64, ppc64le, s390x | -|Ubi-based image | ``nginxcontrib/nginx:1.25.3-ubi``, which is based on ``redhat/ubi9-minimal`` | | ``nginx/nginx-ingress:3.3.2-ubi`` | arm64, amd64, ppc64le, s390x | -{{% /table %}} +{{< bootstrap-table "table table-bordered table-responsive" >}} +|
Name
|
Base image
|
Third-party modules
| DockerHub image | Architectures | +| ---| --- | --- | --- | --- | +|Alpine-based image | ``nginx:1.25.2-alpine``,
based on on ``alpine:3.18`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.1-alpine`` | arm/v7
arm64
amd64
ppc64le
s390x | +|Debian-based image | ``nginx:1.25.2``,
based on on ``debian:12-slim`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.1`` | arm/v7
arm64
amd64
ppc64le
s390x | +|Ubi-based image | ``nginxcontrib/nginx:1.25.2-ubi``,
based on on ``redhat/ubi9-minimal`` | | ``nginx/nginx-ingress:3.2.1-ubi`` | arm64
amd64
ppc64le
s390x | +{{% /bootstrap-table %}} ### Images with NGINX Plus -NGINX Plus images include NGINX Plus R30. +_NGINX Plus images include NGINX Plus R30._ -NGINX Plus images are available through the F5 Container registry `private-registry.nginx.com` - see [Using the NGINX IC Plus JWT token in a Docker Config Secret](/nginx-ingress-controller/installation/using-the-jwt-token-docker-secret) and [Pulling the NGINX Ingress Controller image](/nginx-ingress-controller/installation/pulling-ingress-controller-image). +#### **F5 Container registry** -{{% table %}} -|Name | Base image | Third-party modules | F5 Container Registry Image | Architectures | +NGINX Plus images are available through the F5 Container registry `private-registry.nginx.com` - see [Getting the NGINX Ingress Controller Image with JWT]({{}}) and [Getting the F5 Registry NGINX Ingress Controller Image]({{}}). + +{{< bootstrap-table "table table-striped table-bordered table-responsive" >}} +|
Name
|
Base image
|
Third-party modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.3.2-alpine` | arm64, amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog, FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:3.3.2-alpine-fips` | arm64, amd64 | -|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.3.2` | arm64, amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:3.3.2` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | `nginx-ic-dos/nginx-plus-ingress:3.3.2` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | `nginx-ic-nap-dos/nginx-plus-ingress:3.3.2` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:3.3.2-ubi` | arm64, amd64, s390x | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi8`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:3.3.2-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:3.3.2-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS, NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:3.3.2-ubi` | amd64 | -{{% /table %}} - -We also provide NGINX Plus images through the AWS Marketplace. Please see [Using the AWS Marketplace Ingress Controller Image](/nginx-ingress-controller/installation/using-aws-marketplace-image/) for details on how to set up the required IAM resources in your EKS cluster. - -{{% table %}} -|Name | Base image | Third-party modules | AWS Marketplace Link | Architectures | +|Alpine-based image | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.1-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:3.2.1-alpine-fips` | arm64
amd64 | +|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.1` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:3.2.1` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-dos/nginx-plus-ingress:3.2.1` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.1` | amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:3.2.1-ubi` | arm64
amd64
s390x | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi8`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:3.2.1-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:3.2.1-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.1-ubi` | amd64 | +{{% /bootstrap-table %}} + +#### **AWS Marketplace** + +We also provide NGINX Plus images through the AWS Marketplace. Please see [Using the AWS Marketplace Ingress Controller Image]({{< relref "/installation/nic-images/using-aws-marketplace-image.md" >}}) for details on how to set up the required IAM resources in your EKS cluster. + +{{< bootstrap-table "table table-striped table-bordered table-responsive" >}} +|
Name
|
Base image
|
Third-party modules
| AWS Marketplace Link | Architectures | | ---| ---| --- | --- | --- | -|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | [F5 NGINX Ingress Controller](https://aws.amazon.com/marketplace/pp/prodview-fx3faxl7zqeau) | amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | [F5 NGINX Ingress Controller with F5 NGINX App Protect WAF](https://aws.amazon.com/marketplace/pp/prodview-vnrnxbf6u3nra) | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | [F5 NGINX Ingress Controller with F5 NGINX App Protect WAF and DoS](https://aws.amazon.com/marketplace/pp/prodview-yltaqwzwrnhco) | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | [F5 NGINX Ingress Controller with F5 NGINX App Protect DoS](https://aws.amazon.com/marketplace/pp/prodview-sghjw2csktega) | amd64 | -{{% /table %}} +|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | [F5 NGINX Ingress Controller](https://aws.amazon.com/marketplace/pp/prodview-fx3faxl7zqeau) | amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | [F5 NGINX Ingress Controller with F5 NGINX App Protect WAF](https://aws.amazon.com/marketplace/pp/prodview-vnrnxbf6u3nra) | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | [F5 NGINX Ingress Controller with F5 NGINX App Protect WAF and DoS](https://aws.amazon.com/marketplace/pp/prodview-yltaqwzwrnhco) | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | [F5 NGINX Ingress Controller with F5 NGINX App Protect DoS](https://aws.amazon.com/marketplace/pp/prodview-sghjw2csktega) | amd64 | +{{% /bootstrap-table %}} -We also provide NGINX Plus images through the Google Cloud Marketplace. Please see [Using GCP Marketplace Ingress Controller](/nginx-ingress-controller/installation/using-gcp-marketplace-package/) for details on how use them. +#### **Google Cloud Marketplace** +We also provide NGINX Plus images through the Google Cloud Marketplace. Please see [Using the GCP Marketplace NGINX Ingress Controller Image]({{< relref "/installation/nic-images/using-gcp-marketplace-package.md" >}}) for details on how to use them. -{{% table %}} -|Name | Base image | Third-party modules | GCP Marketplace Link | Architectures | +{{< bootstrap-table "table table-striped table-bordered table-responsive" >}} +|
Name
|
Base image
|
Third-party modules
| GCP Marketplace Link | Architectures | | ---| ---| --- | --- | --- | -|Debian-based image | ``debian:11-slim`` | NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | [F5 NGINX Ingress Controller](https://console.cloud.google.com/marketplace/product/f5-7626-networks-public/nginx-ingress-plus) | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS, NGINX Plus JavaScript and OpenTracing modules, OpenTracing tracers for Jaeger, Zipkin and Datadog | [F5 NGINX Ingress Controller w/ F5 NGINX App Protect DoS](https://console.cloud.google.com/marketplace/product/f5-7626-networks-public/nginx-ingress-plus-dos) | amd64 | -{{% /table %}} +|Debian-based image | ``debian:11-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | [F5 NGINX Ingress Controller](https://console.cloud.google.com/marketplace/product/f5-7626-networks-public/nginx-ingress-plus) | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | [F5 NGINX Ingress Controller w/ F5 NGINX App Protect DoS](https://console.cloud.google.com/marketplace/product/f5-7626-networks-public/nginx-ingress-plus-dos) | amd64 | +{{% /bootstrap-table %}} ### Custom Images @@ -107,8 +111,4 @@ You can customize an existing Dockerfile or use it as a reference to create a ne ## Supported Helm Versions -The Ingress Controller supports installation via Helm 3.0+. - -## Recommended Hardware - -See the [Sizing guide](https://www.nginx.com/resources/datasheets/nginx-ingress-controller-kubernetes-sizing-guide/) for recommendations. +NGINX Ingress Controller can be [installed]({{< relref "/installation/installing-nic/installation-with-helm.md" >}}) using Helm 3.0 or later. diff --git a/docs/content/third-party-modules/_index.md b/docs/content/third-party-modules/_index.md deleted file mode 100644 index 77b0c5e387b..00000000000 --- a/docs/content/third-party-modules/_index.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: Third Party Modules -description: -weight: 1700 -menu: - docs: - parent: NGINX Ingress Controller ---- diff --git a/docs/content/third-party-modules/opentracing.md b/docs/content/third-party-modules/opentracing.md deleted file mode 100644 index 83de4ff35b7..00000000000 --- a/docs/content/third-party-modules/opentracing.md +++ /dev/null @@ -1,103 +0,0 @@ ---- -title: OpenTracing -description: "This document explains how to use OpenTracing with the Ingress Controller." -weight: 2000 -doctypes: [""] -aliases: - - /opentracing/ -toc: true -docs: "DOCS-618" ---- - - -The Ingress Controller supports [OpenTracing](https://opentracing.io/) with the third-party module [opentracing-contrib/nginx-opentracing](https://github.com/opentracing-contrib/nginx-opentracing). - -This document explains how to use OpenTracing with the Ingress Controller. - -**Note**: The examples below use the snippets annotations, which are disabled by default. To use snippets, set the [`enable-snippets`](/nginx-ingress-controller/configuration/global-configuration/command-line-arguments#cmdoption-enable-snippets) command-line argument. - -## Prerequisites - -1. **Use an Ingress Controller image that contains OpenTracing.** - - You can find the images that include OpenTracing listed [in the technical specs doc]({{< relref "technical-specifications.md#supported-docker-images" >}}). - - Alternatively, you can [build your own image]({{< relref "installation/building-ingress-controller-image.md" >}}) using `debian-image` (or `alpine-image`) for NGINX or `debian-image-plus` (or `alpine-image-plus`) for NGINX Plus. -[Jaeger](https://github.com/jaegertracing/jaeger-client-cpp), [Zipkin](https://github.com/rnburn/zipkin-cpp-opentracing) and [Datadog](https://github.com/DataDog/dd-opentracing-cpp/) tracers are installed by default. - -2. **Load the OpenTracing module.** You need to load the module with the configuration for the chosen tracer using the following ConfigMap keys: - - `opentracing-tracer`: sets the path to the vendor tracer binary plugin. This is the path you used in the COPY line of step *ii* above. - - `opentracing-tracer-config`: sets the tracer configuration in JSON format. - - Below an example on how to use those keys to load the module with Jaeger tracer: - - ```yaml - opentracing-tracer: "/usr/local/lib/libjaegertracing_plugin.so" - opentracing-tracer-config: | - { - "service_name": "nginx-ingress", - "propagation_format": "w3c", - "sampler": { - "type": "const", - "param": 1 - }, - "reporter": { - "localAgentHostPort": "jaeger-agent.default.svc.cluster.local:6831" - } - } - ``` - -## Enable OpenTracing Globally - -To enable OpenTracing globally (for all Ingress, VirtualServer and VirtualServerRoute resources), set the `opentracing` ConfigMap key to `True`: - -```yaml -opentracing: True -``` - -## Enable/Disable OpenTracing per Ingress Resource - -It is possible to use annotations to enable or disable OpenTracing for a specific Ingress Resource. As mentioned in the prerequisites section, both `opentracing-tracer` and `opentracing-tracer-config` must be configured. - -Consider the following two cases: - -1. OpenTracing is globally disabled. - 1. To enable OpenTracing for a specific Ingress Resource, use the server snippet annotation: - - ```yaml - nginx.org/server-snippets: | - opentracing on; - ``` - - 1. To enable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration) and (2) use the location snippets annotation to enable OpenTracing for the paths of a specific Minion Ingress resource: - - ```yaml - nginx.org/location-snippets: | - opentracing on; - ``` - -2. OpenTracing is globally enabled: - 1. To disable OpenTracing for a specific Ingress Resource, use the server snippet annotation: - - ```yaml - nginx.org/server-snippets: | - opentracing off; - ``` - - 1. To disable OpenTracing for specific paths, (1) you need to use [Mergeable Ingress resources](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration) and (2) use the location snippets annotation to disable OpenTracing for the paths of a specific Minion Ingress resource: - - ```yaml - nginx.org/location-snippets: | - opentracing off; - ``` - -## Customize OpenTracing - -You can customize OpenTracing though the supported [OpenTracing module directives](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md). Use the snippets ConfigMap keys or annotations to insert those directives into the http, server or location contexts of the generated NGINX configuration. - -For example, to propagate the active span context for upstream requests, it is required to set the `opentracing_propagate_context` directive, which you can add to an Ingress resource using the location snippets annotation: - -```yaml -nginx.org/location-snippets: | - opentracing_propagate_context; -``` - -**Note**: `opentracing_propagate_context` and `opentracing_grpc_propagate_context` directives can be used in http, server or location contexts according to the [module documentation](https://github.com/opentracing-contrib/nginx-opentracing/blob/master/doc/Reference.md#opentracing_propagate_context). However, because of the way the module works and how the Ingress Controller generates the NGINX configuration, it is only possible to use the directive in the location context. diff --git a/docs/content/troubleshooting/troubleshoot-common.md b/docs/content/troubleshooting/troubleshoot-common.md index 63ff8f31e81..fd704dd385b 100644 --- a/docs/content/troubleshooting/troubleshoot-common.md +++ b/docs/content/troubleshooting/troubleshoot-common.md @@ -145,7 +145,7 @@ controller: nginxplus: plus image: repository: nginx/nginx-ingress - tag: 3.3.0 + tag: 2.4.1 # NGINX Configmap config: entries: diff --git a/docs/content/tutorials/custom-listen-ports.md b/docs/content/tutorials/custom-listen-ports.md index b7fcac13ca3..c11297f6f07 100644 --- a/docs/content/tutorials/custom-listen-ports.md +++ b/docs/content/tutorials/custom-listen-ports.md @@ -88,7 +88,7 @@ spec: spec: serviceAccountName: nginx-ingress containers: - - image: nginx/nginx-ingress:3.3.0 + - image: nginx/nginx-ingress:3.2.0 imagePullPolicy: IfNotPresent name: nginx-ingress ports: diff --git a/docs/content/tutorials/oidc-custom-configuration.md b/docs/content/tutorials/oidc-custom-configuration.md index 4f79bb0d8ec..2604cead1a7 100644 --- a/docs/content/tutorials/oidc-custom-configuration.md +++ b/docs/content/tutorials/oidc-custom-configuration.md @@ -30,7 +30,7 @@ Run the below command to generate a ConfigMap with the contents of the `oidc.con **NOTE** The ConfigMap must be deployed in the same `namespace` as the F5 NGINX Ingress Controller. ```console -kubectl create configmap oidc-config-map --from-literal=oidc.conf="$(curl -k https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/internal/configs/oidc/oidc.conf)" +kubectl create configmap oidc-config-map --from-literal=oidc.conf="$(curl -k https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/internal/configs/oidc/oidc.conf)" ``` Use the `kubectl describe` command to confirm the contents of the ConfigMap are correct. diff --git a/docs/content/tutorials/virtual-server-with-custom-listener-ports.md b/docs/content/tutorials/virtual-server-with-custom-listener-ports.md index a0badbe88fc..92dc50c9dd4 100644 --- a/docs/content/tutorials/virtual-server-with-custom-listener-ports.md +++ b/docs/content/tutorials/virtual-server-with-custom-listener-ports.md @@ -15,7 +15,7 @@ Each field must reference a valid listener defined by in a [GlobalConfiguration] 1. Create a yaml file called `nginx-configuration.yaml` with the below content: ```yaml -apiVersion: k8s.nginx.org/v1 +apiVersion: k8s.nginx.org/v1alpha1 kind: GlobalConfiguration metadata: name: nginx-configuration diff --git a/docs/content/usage-reporting.md b/docs/content/usage-reporting.md index 5453fd56a73..4a6d6f60232 100644 --- a/docs/content/usage-reporting.md +++ b/docs/content/usage-reporting.md @@ -87,7 +87,7 @@ To make the credential available to Usage Reporting, we need to create a Kuberne If you need to update the basic-auth credentials for NGINX Management Suite in the future, update the `username` and `password` fields, and apply the changes by running the command again. Usage Reporting will automatically detect the changes, using the new username and password without redeployment. -5. Download and save the deployment file [cluster-connector.yaml](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.0/examples/shared-examples/usage-reporting/cluster-connector.yaml). Edit the following under the `args` section and then save the file: +5. Download and save the deployment file [cluster-connector.yaml](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.1.1/examples/shared-examples/usage-reporting/cluster-connector.yaml). Edit the following under the `args` section and then save the file: ```yaml args: diff --git a/docs/layouts/shortcodes/call-out.html b/docs/layouts/shortcodes/call-out.html new file mode 100644 index 00000000000..d8e591d8322 --- /dev/null +++ b/docs/layouts/shortcodes/call-out.html @@ -0,0 +1,3 @@ +
+
{{ .Get 1 }}
{{ .Inner | markdownify }}
+
diff --git a/docs/layouts/shortcodes/custom-styles.html b/docs/layouts/shortcodes/custom-styles.html new file mode 100644 index 00000000000..8ea60c48b66 --- /dev/null +++ b/docs/layouts/shortcodes/custom-styles.html @@ -0,0 +1,41 @@ + From f7cf0e2b25cdc54cb07fa31490aef96f97fbf8b2 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 11:31:49 +0000 Subject: [PATCH 02/17] Fix release numbers for revamp branch --- .../configuration/configuration-examples.md | 4 +- .../configmap-resource.md | 16 +-- .../global-configuration/custom-templates.md | 2 +- .../handling-host-and-listener-collisions.md | 2 +- ...advanced-configuration-with-annotations.md | 38 +++--- .../ingress-resources/basic-configuration.md | 2 +- .../cross-namespace-configuration.md | 4 +- .../ingress-resources/custom-annotations.md | 4 +- docs/content/configuration/policy-resource.md | 4 +- .../configuration/transportserver-resource.md | 2 +- ...server-and-virtualserverroute-resources.md | 6 +- .../building-nginx-ingress-controller.md | 4 +- .../installing-nic/installation-with-helm.md | 2 +- .../installation-with-manifests.md | 2 +- .../installation-with-operator.md | 2 +- .../app-protect-dos/configuration.md | 2 +- .../app-protect-dos/installation.md | 6 +- .../app-protect-waf/configuration.md | 24 ++-- .../app-protect-waf/installation.md | 4 +- .../pulling-ingress-controller-image.md | 18 +-- .../using-the-jwt-token-docker-secret.md | 14 +-- .../content/overview/controller-comparison.md | 4 +- docs/content/overview/nginx-plus.md | 6 +- docs/content/releases.md | 118 ++++++++++++++++++ docs/content/technical-specifications.md | 28 ++--- .../tutorials/oidc-custom-configuration.md | 2 +- 26 files changed, 219 insertions(+), 101 deletions(-) diff --git a/docs/content/configuration/configuration-examples.md b/docs/content/configuration/configuration-examples.md index 181461df68c..a4b30ac51eb 100644 --- a/docs/content/configuration/configuration-examples.md +++ b/docs/content/configuration/configuration-examples.md @@ -11,5 +11,5 @@ docs: "DOCS-584" Our [GitHub repo](https://github.com/nginxinc/kubernetes-ingress) includes a number of configuration examples: -- [*Examples of Custom Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. -- [*Examples of Ingress Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. +- [*Examples of Custom Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. +- [*Examples of Ingress Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. diff --git a/docs/content/configuration/global-configuration/configmap-resource.md b/docs/content/configuration/global-configuration/configmap-resource.md index ca3ced6b44b..c8050a04505 100644 --- a/docs/content/configuration/global-configuration/configmap-resource.md +++ b/docs/content/configuration/global-configuration/configmap-resource.md @@ -91,10 +91,10 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``server-names-hash-max-size`` | Sets the value of the [server_names_hash_max_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) directive. | ``1024`` | | |``map-hash-bucket-size`` | Sets the value of the [map_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size) directive.| ``256`` | | |``map-hash-max-size`` | Sets the value of the [map_hash_max_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_max_size) directive. | ``2048`` | | -|``resolver-addresses`` | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, ``kube-dns.kube-system.svc.cluster.local`` ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | -|``resolver-ipv6`` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | ``True`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | -|``resolver-valid`` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | -|``resolver-timeout`` | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | ``30s`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services). | +|``resolver-addresses`` | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, ``kube-dns.kube-system.svc.cluster.local`` ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | +|``resolver-ipv6`` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | ``True`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | +|``resolver-valid`` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | +|``resolver-timeout`` | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | ``30s`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | |``keepalive-timeout`` | Sets the value of the [keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) directive. | ``65s`` | | |``keepalive-requests`` | Sets the value of the [keepalive_requests](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) directive. | ``100`` | | |``variables-hash-bucket-size`` | Sets the value of the [variables_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_bucket_size) directive. | ``256`` | | @@ -109,9 +109,9 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``error-log-level`` | Sets the global [error log level](https://nginx.org/en/docs/ngx_core_module.html#error_log) for NGINX. | ``notice`` | | |``access-log-off`` | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). | ``False`` | | |``default-server-access-log-off`` | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) for the default server. If access log is disabled globally (``access-log-off: "True"``), then the default server access log is always disabled. | ``False`` | | -|``log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/shared-examples/custom-log-format). | +|``log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/shared-examples/custom-log-format). | |``log-format-escaping`` | Sets the characters escaping for the variables of the log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | -|``stream-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx.tmpl). | | +|``stream-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx.tmpl). | | |``stream-log-format-escaping`` | Sets the characters escaping for the variables of the stream log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | {{% /table %}} @@ -147,7 +147,7 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |ConfigMap Key | Description | Default | Example | | ---| ---| ---| --- | |``http2`` | Enables HTTP/2 in servers with SSL enabled. | ``False`` | | -|``proxy-protocol`` | Enables PROXY Protocol for incoming connections. | ``False`` | [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/shared-examples/proxy-protocol). | +|``proxy-protocol`` | Enables PROXY Protocol for incoming connections. | ``False`` | [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/shared-examples/proxy-protocol). | {{% /table %}} ### Backend Services (Upstreams) @@ -171,7 +171,7 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``http-snippets`` | Sets a custom snippet in http context. | N/A | | |``location-snippets`` | Sets a custom snippet in location context. | N/A | | |``server-snippets`` | Sets a custom snippet in server context. | N/A | | -|``stream-snippets`` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/tcp-udp). | +|``stream-snippets`` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/tcp-udp). | |``main-template`` | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | |``ingress-template`` | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | |``virtualserver-template`` | Sets the NGINX configuration template for an VirtualServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | diff --git a/docs/content/configuration/global-configuration/custom-templates.md b/docs/content/configuration/global-configuration/custom-templates.md index 35b70302a89..6ff302a8eca 100644 --- a/docs/content/configuration/global-configuration/custom-templates.md +++ b/docs/content/configuration/global-configuration/custom-templates.md @@ -9,4 +9,4 @@ docs: "DOCS-587" --- -The Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/shared-examples/custom-templates). +The Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/shared-examples/custom-templates). diff --git a/docs/content/configuration/handling-host-and-listener-collisions.md b/docs/content/configuration/handling-host-and-listener-collisions.md index 483544408f8..2975acd2d6b 100644 --- a/docs/content/configuration/handling-host-and-listener-collisions.md +++ b/docs/content/configuration/handling-host-and-listener-collisions.md @@ -87,7 +87,7 @@ Similarly, if `cafe-ingress` was created first, it will win `cafe.example.com` a It is possible to merge configuration for multiple Ingress resources for the same host. One common use case for this approach is distributing resources across multiple namespaces. See the [Cross-namespace Configuration](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration/) doc for more information. -It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/cross-namespace-configuration) on GitHub. +It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/cross-namespace-configuration) on GitHub. It is *not* possible to merge configuration for multiple TransportServer resources. diff --git a/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md b/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md index ff71d698680..07d3c4cc545 100644 --- a/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md +++ b/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md @@ -122,7 +122,7 @@ The table below summarizes the available annotations. | ---| ---| ---| ---| --- | |``nginx.org/proxy-hide-headers`` | ``proxy-hide-headers`` | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: ``"nginx.org/proxy-hide-headers": "header-a,header-b"`` | N/A | | |``nginx.org/proxy-pass-headers`` | ``proxy-pass-headers`` | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: ``"nginx.org/proxy-pass-headers": "header-a,header-b"`` | N/A | | -|``nginx.org/rewrites`` | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [Rewrites Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/rewrites). | +|``nginx.org/rewrites`` | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [Rewrites Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/rewrites). | {{% /table %}} ### Auth and SSL/TLS @@ -138,10 +138,10 @@ The table below summarizes the available annotations. |``nginx.org/hsts-behind-proxy`` | ``hsts-behind-proxy`` | Enables HSTS based on the value of the ``http_x_forwarded_proto`` request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of the Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the ``nginx.org/redirect-to-https`` annotation. | ``False`` | | |``nginx.org/basic-auth-secret`` | N/A | Specifies a Secret resource with a user list for HTTP Basic authentication. | N/A | | |``nginx.org/basic-auth-realm`` | N/A | Specifies a realm. | N/A | | -|``nginx.com/jwt-key`` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | -|``nginx.com/jwt-realm`` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | -|``nginx.com/jwt-token`` | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization`` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | -|``nginx.com/jwt-login-url`` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt). | +|``nginx.com/jwt-key`` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-realm`` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-token`` | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization`` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-login-url`` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | {{% /table %}} ### Listeners @@ -159,19 +159,19 @@ The table below summarizes the available annotations. |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | |``nginx.org/lb-method`` | ``lb-method`` | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``"round_robin"``. | ``"random two least_conn"`` | | -|``nginx.org/ssl-services`` | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [SSL Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/ssl-services). | -|``nginx.org/grpc-services`` | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2`` ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [GRPC Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/grpc-services). | -|``nginx.org/websocket-services`` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/websocket). | +|``nginx.org/ssl-services`` | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [SSL Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/ssl-services). | +|``nginx.org/grpc-services`` | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2`` ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [GRPC Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/grpc-services). | +|``nginx.org/websocket-services`` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/websocket). | |``nginx.org/max-fails`` | ``max-fails`` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the ``server`` directive. | ``1`` | | |``nginx.org/max-conns`` | N\A | Sets the value of the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the ``server`` directive. | ``0`` | | |``nginx.org/upstream-zone-size`` | ``upstream-zone-size`` | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | ``256K`` | | |``nginx.org/fail-timeout`` | ``fail-timeout`` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the ``server`` directive. | ``10s`` | | -|``nginx.com/sticky-cookie-services`` | N/A | Configures session persistence. | N/A | [Session Persistence](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/session-persistence). | +|``nginx.com/sticky-cookie-services`` | N/A | Configures session persistence. | N/A | [Session Persistence](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/session-persistence). | |``nginx.org/keepalive`` | ``keepalive`` | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that ``proxy_set_header Connection "";`` is added to the generated configuration when the value > 0. | ``0`` | | -|``nginx.com/health-checks`` | N/A | Enables active health checks. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | -|``nginx.com/health-checks-mandatory`` | N/A | Configures active health checks as mandatory. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | -|``nginx.com/health-checks-mandatory-queue`` | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | ``0`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | -|``nginx.com/slow-start`` | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks). | ``"0s"`` | | +|``nginx.com/health-checks`` | N/A | Enables active health checks. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | +|``nginx.com/health-checks-mandatory`` | N/A | Configures active health checks as mandatory. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | +|``nginx.com/health-checks-mandatory-queue`` | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | ``0`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | +|``nginx.com/slow-start`` | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | ``"0s"`` | | {{% /table %}} ### Snippets and Custom Templates @@ -190,11 +190,11 @@ The table below summarizes the available annotations. {{% table %}} |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | -|``appprotect.f5.com/app-protect-policy`` | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable`` is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-enable`` | N/A | Enable App Protect for the Ingress Resource. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log-enable`` | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log`` | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log-destination`` | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | ``syslog:server=localhost:514`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-policy`` | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable`` is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-enable`` | N/A | Enable App Protect for the Ingress Resource. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log-enable`` | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log`` | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log-destination`` | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | ``syslog:server=localhost:514`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | {{% /table %}} ### App Protect DoS @@ -204,5 +204,5 @@ The table below summarizes the available annotations. {{% table %}} |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | -|``appprotectdos.f5.com/app-protect-dos-resource`` | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource](/nginx-ingress-controller/app-protect-dos/dos-protected/). | N/A | [Example for App Protect DoS](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-dos). | +|``appprotectdos.f5.com/app-protect-dos-resource`` | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource](/nginx-ingress-controller/app-protect-dos/dos-protected/). | N/A | [Example for App Protect DoS](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-dos). | {{% /table %}} diff --git a/docs/content/configuration/ingress-resources/basic-configuration.md b/docs/content/configuration/ingress-resources/basic-configuration.md index 6aba7b31d3f..f184609361a 100644 --- a/docs/content/configuration/ingress-resources/basic-configuration.md +++ b/docs/content/configuration/ingress-resources/basic-configuration.md @@ -53,7 +53,7 @@ Here is a breakdown of what this Ingress resource definition means: - The rule with the path `/coffee` instructs NGINX to distribute the requests with the `/coffee` URI among the pods of the *coffee* service, which is deployed with the name `coffee‑svc` in the cluster. - Both rules instruct NGINX to distribute the requests to `port 80` of the corresponding service (the `servicePort` field). -> For complete instructions on deploying the Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/complete-example) in our GitHub repository. +> For complete instructions on deploying the Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/complete-example) in our GitHub repository. > To learn more about the Ingress resource, see the [Ingress resource documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) in the Kubernetes docs. diff --git a/docs/content/configuration/ingress-resources/cross-namespace-configuration.md b/docs/content/configuration/ingress-resources/cross-namespace-configuration.md index 3edb02ad58f..69aa78e51fc 100644 --- a/docs/content/configuration/ingress-resources/cross-namespace-configuration.md +++ b/docs/content/configuration/ingress-resources/cross-namespace-configuration.md @@ -9,6 +9,6 @@ docs: "DOCS-594" --- -You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. +You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. -As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. +As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. diff --git a/docs/content/configuration/ingress-resources/custom-annotations.md b/docs/content/configuration/ingress-resources/custom-annotations.md index 683114c98c0..68c927a0e66 100644 --- a/docs/content/configuration/ingress-resources/custom-annotations.md +++ b/docs/content/configuration/ingress-resources/custom-annotations.md @@ -23,7 +23,7 @@ Custom annotations allow you to add an annotation for an NGINX feature that is n ## Usage -The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/internal/configs/version1/nginx-plus.ingress.tmpl). +The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx-plus.ingress.tmpl). To support custom annotations, the template has access to the information about the Ingress resource - its *name*, *namespace* and *annotations*. It is possible to check if a particular annotation present in the Ingress resource and conditionally insert NGINX configuration directives at multiple NGINX contexts - `http`, `server`, `location` or `upstream`. Additionally, you can get the value that is set to the annotation. @@ -139,4 +139,4 @@ deny all; ## Example -See the [custom annotations example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/ingress-resources/custom-annotations). +See the [custom annotations example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/ingress-resources/custom-annotations). diff --git a/docs/content/configuration/policy-resource.md b/docs/content/configuration/policy-resource.md index 6fc4c512359..bc8c0017dfc 100644 --- a/docs/content/configuration/policy-resource.md +++ b/docs/content/configuration/policy-resource.md @@ -12,7 +12,7 @@ The Policy resource allows you to configure features like access control and rat The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/access-control). +This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/access-control). ## Prerequisites @@ -459,7 +459,7 @@ NGINX Plus will pass the ID of an authenticated user to the backend in the HTTP #### Prerequisites In order to use OIDC, you need to enable [zone synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync/). If you don't set up zone synchronization, NGINX Plus will fail to reload. -You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). +You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). > **Note**: The configuration in the example doesn't enable TLS and the synchronization between the replica happens in clear text. This could lead to the exposure of tokens. diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index 62ae490d043..ae0b0295c9f 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -10,7 +10,7 @@ docs: "DOCS-598" The TransportServer resource allows you to configure TCP, UDP, and TLS Passthrough load balancing. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the TransportServer resource. To see additional examples of using the resource for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources) folder in our GitHub repo. +This document is the reference documentation for the TransportServer resource. To see additional examples of using the resource for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources) folder in our GitHub repo. ## Prerequisites diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md index d9bb7c6ef91..fc7ff34cb5c 100644 --- a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md +++ b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md @@ -12,7 +12,7 @@ docs: "DOCS-599" The VirtualServer and VirtualServerRoute resources, introduced in release 1.5, enable use cases not supported with the Ingress resource, such as traffic splitting and advanced content-based routing. The resources are implemented as [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the resources. To see additional examples of using the resources for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources) folder in our GitHub repo. +This document is the reference documentation for the resources. To see additional examples of using the resources for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources) folder in our GitHub repo. ## VirtualServer Specification @@ -346,7 +346,7 @@ tls: |Field | Description | Type | Required | | ---| ---| ---| --- | |``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | -|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | |``subselector`` | Selects the pods within the service using label keys and values. By default, all pods of the service are selected. Note: the specified labels are expected to be present in the pods when they are created. If the pod labels are updated, the Ingress Controller will not see that change until the number of the pods is changed. | ``map[string]string`` | No | |``use-cluster-ip`` | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method`` and ``next-upstream``) will have no effect, as the Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | ``boolean`` | No | |``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. The port must fall into the range ``1..65535``. | ``uint16`` | Yes | @@ -633,7 +633,7 @@ proxy: |``upstream`` | The name of the upstream which the requests will be proxied to. The upstream with that name must be defined in the resource. | ``string`` | Yes | |``requestHeaders`` | The request headers modifications. | [action.Proxy.RequestHeaders](#actionproxyrequestheaders) | No | |``responseHeaders`` | The response headers modifications. | [action.Proxy.ResponseHeaders](#actionproxyresponseheaders) | No | -|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/rewrites) example. | ``string`` | No | +|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/rewrites) example. | ``string`` | No | {{% /table %}} ### Action.Proxy.RequestHeaders diff --git a/docs/content/installation/building-nginx-ingress-controller.md b/docs/content/installation/building-nginx-ingress-controller.md index a3d6374e6d0..011d896e257 100644 --- a/docs/content/installation/building-nginx-ingress-controller.md +++ b/docs/content/installation/building-nginx-ingress-controller.md @@ -41,10 +41,10 @@ Get your system ready for building and pushing the NGINX Ingress Controller imag cd kubernetes-ingress ``` - For instance if you want to clone version v3.2.1, the commands to run would be: + For instance if you want to clone version v3.2.2, the commands to run would be: ```shell - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.2 cd kubernetes-ingress ``` diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index 42f12b572dc..bc486464249 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -309,7 +309,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **controller.logLevel** | The log level of the Ingress Controller. | 1 | | **controller.image.digest** | The image digest of the Ingress Controller. | None | | **controller.image.repository** | The image repository of the Ingress Controller. | nginx/nginx-ingress | -| **controller.image.tag** | The tag of the Ingress Controller image. | 3.2.1 | +| **controller.image.tag** | The tag of the Ingress Controller image. | 3.2.2 | | **controller.image.pullPolicy** | The pull policy for the Ingress Controller image. | IfNotPresent | | **controller.lifecycle** | The lifecycle of the Ingress Controller pods. | {} | | **controller.customConfigMap** | The name of the custom ConfigMap used by the Ingress Controller. If set, then the default config is ignored. | "" | diff --git a/docs/content/installation/installing-nic/installation-with-manifests.md b/docs/content/installation/installing-nic/installation-with-manifests.md index 21d9507574c..a9165071259 100644 --- a/docs/content/installation/installing-nic/installation-with-manifests.md +++ b/docs/content/installation/installing-nic/installation-with-manifests.md @@ -36,7 +36,7 @@ git clone https://github.com/nginxinc/kubernetes-ingress.git --branch Check out the complete [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-dos). +> Check out the complete [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-dos). ## App Protect DoS Configuration diff --git a/docs/content/installation/integrations/app-protect-dos/installation.md b/docs/content/installation/integrations/app-protect-dos/installation.md index 700f330f900..1b77dde38b7 100644 --- a/docs/content/installation/integrations/app-protect-dos/installation.md +++ b/docs/content/installation/integrations/app-protect-dos/installation.md @@ -34,10 +34,10 @@ Get your system ready for building and pushing the NGINX Ingress Controller imag cd kubernetes-ingress ``` - For instance if you want to clone version v3.2.1, the commands to run would be: + For instance if you want to clone version v3.2.2, the commands to run would be: ```shell - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.2 cd kubernetes-ingress/deployments ``` @@ -177,7 +177,7 @@ To enable the NGINX App Protect DoS Module: {{< include "installation/manifests/verify-pods-are-running.md" >}} -For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-dos/configuration.md" >}}),the [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-dos). +For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-dos/configuration.md" >}}),the [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-dos). --- diff --git a/docs/content/installation/integrations/app-protect-waf/configuration.md b/docs/content/installation/integrations/app-protect-waf/configuration.md index 35577695265..74b9d8af2d8 100644 --- a/docs/content/installation/integrations/app-protect-waf/configuration.md +++ b/docs/content/installation/integrations/app-protect-waf/configuration.md @@ -8,7 +8,7 @@ docs: "DOCS-578" aliases: ["/app-protect/configuration/"] --- -> Check out the complete NGINX Ingress Controller with NGINX App Protect WAF example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf). +> Check out the complete NGINX Ingress Controller with NGINX App Protect WAF example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). ## Global Configuration @@ -245,8 +245,8 @@ These are the typical steps to deploy an OpenAPI protection Policy in NGINX Ingr 3. Make other custom changes if needed (e.g. enable Data Guard protection). 4. Use a tool to convert the result to YAML. There are many, for example: [`yq` utility](https://github.com/mikefarah/yq). 5. Add the YAML properties to create an `APPolicy` Custom Resource putting the policy itself (as in step 4) within the `spec` property of the Custom Resource. Refer to the [NGINX App Protect Policies](#nginx-app-protect-waf-policies) section above. -6. Create a `Policy` object which references the `APPolicy` Custom Resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/app-protect-waf/waf.yaml). -7. Finally, attach the `Policy` object to a `VirtualServer` resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.1/examples/custom-resources/app-protect-waf/virtual-server.yaml). +6. Create a `Policy` object which references the `APPolicy` Custom Resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/app-protect-waf/waf.yaml). +7. Finally, attach the `Policy` object to a `VirtualServer` resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/app-protect-waf/virtual-server.yaml). **Note**: You need to make sure that the server where the resource files are located is always available when you are compiling your policy. @@ -381,7 +381,7 @@ The `link` option is also available in the `openApiFileReference` property and i In this example we deploy NGINX Ingress Controller with NGINX Plus and NGINX App Protect WAF, deploy a simple web application, and then configure load balancing and WAF protection for that application using the VirtualServer resource. -**Note:** You can find the example, and the files referenced, on [GitHub](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-waf). +**Note:** You can find the example, and the files referenced, on [GitHub](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-waf). ## Prerequisites @@ -403,7 +403,7 @@ In this example we deploy NGINX Ingress Controller with NGINX Plus and NGINX App Create the application deployment and service: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/webapp.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/webapp.yaml ``` ### Step 2. Deploy the AP Policy @@ -411,15 +411,15 @@ Create the application deployment and service: 1. Create the syslog service and pod for the NGINX App Protect WAF security logs: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/syslog.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/syslog.yaml ``` 2. Create the User-Defined Signature, WAF policy, and log configuration: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/ap-apple-uds.yaml - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/ap-dataguard-alarm-policy.yaml - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/ap-logconf.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/ap-apple-uds.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/ap-dataguard-alarm-policy.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/ap-logconf.yaml ``` ### Step 3 - Deploy the WAF Policy @@ -427,7 +427,7 @@ Create the application deployment and service: Create the WAF policy ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/waf.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/waf.yaml ``` Note the NGINX App Protect WAF configuration settings in the Policy resource. They enable WAF protection by configuring NGINX App Protect WAF with the policy and log configuration created in the previous step. @@ -437,7 +437,7 @@ Create the WAF policy 1. Create the VirtualServer Resource: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/virtual-server.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/virtual-server.yaml ``` Note that the VirtualServer references the policy waf-policy created in Step 3. @@ -478,7 +478,7 @@ To access the application, curl the coffee and the tea services. We'll use the - ### Configuration Example of Virtual Server -Refer to GitHub repo for [Virtual Server example](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/examples/custom-resources/app-protect-waf/webapp.yaml). +Refer to GitHub repo for [Virtual Server example](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/webapp.yaml). ```yaml apiVersion: k8s.nginx.org/v1 diff --git a/docs/content/installation/integrations/app-protect-waf/installation.md b/docs/content/installation/integrations/app-protect-waf/installation.md index a29038920de..18ab6b27523 100644 --- a/docs/content/installation/integrations/app-protect-waf/installation.md +++ b/docs/content/installation/integrations/app-protect-waf/installation.md @@ -23,7 +23,7 @@ You can also [install the Ingress Controller with App Protect WAF by using Helm] 2. Clone the Ingress Controller repo: ```console - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.1 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.2 cd kubernetes-ingress ``` @@ -65,4 +65,4 @@ Take the steps below to set up and deploy the NGINX Ingress Controller and App P 3. Enable the App Protect WAF module by adding the `enable-app-protect` [cli argument]({{< relref "configuration/global-configuration/command-line-arguments.md#cmdoption-enable-app-protect" >}}) to your Deployment or DaemonSet file. 4. [Deploy the Ingress Controller]({{< relref "installation/installing-nic/installation-with-manifests.md#deploy-ingress-controller" >}}). -For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-waf/configuration.md" >}}) and the NGINX Ingress Controller with App Protect example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/app-protect-waf" >}}). +For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-waf/configuration.md" >}}) and the NGINX Ingress Controller with App Protect example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf" >}}). diff --git a/docs/content/installation/nic-images/pulling-ingress-controller-image.md b/docs/content/installation/nic-images/pulling-ingress-controller-image.md index 5abdf42f929..d34fd1239d1 100644 --- a/docs/content/installation/nic-images/pulling-ingress-controller-image.md +++ b/docs/content/installation/nic-images/pulling-ingress-controller-image.md @@ -38,7 +38,7 @@ The steps provided are for Linux. For Mac or Windows, consult the [Docker for Ma Next, pull the image you need from `private-registry.nginx.com`. To find the correct image, consult the [Tech Specs guide]({{< relref "technical-specifications#images-with-nginx-plus" >}}). -To pull an image, follow these steps. Replace `` with the specific version you need, for example, `3.2.1`. +To pull an image, follow these steps. Replace `` with the specific version you need, for example, `3.2.2`. - For NGINX Plus Ingress Controller, run: @@ -65,9 +65,9 @@ $ curl https://private-registry.nginx.com/v2/nginx-ic/nginx-plus-ingress/tags/li { "name": "nginx-ic/nginx-plus-ingress", "tags": [ - "3.2.1-alpine", - "3.2.1-ubi", - "3.2.1" + "3.2.2-alpine", + "3.2.2-ubi", + "3.2.2" ] } @@ -75,8 +75,8 @@ $ curl https://private-registry.nginx.com/v2/nginx-ic-nap/nginx-plus-ingress/tag { "name": "nginx-ic-nap/nginx-plus-ingress", "tags": [ - "3.2.1-ubi", - "3.2.1" + "3.2.2-ubi", + "3.2.2" ] } @@ -84,8 +84,8 @@ $ curl https://private-registry.nginx.com/v2/nginx-ic-dos/nginx-plus-ingress/tag { "name": "nginx-ic-dos/nginx-plus-ingress", "tags": [ - "3.2.1-ubi", - "3.2.1" + "3.2.2-ubi", + "3.2.2" ] } ``` @@ -102,7 +102,7 @@ After pulling the image, tag it and upload it to your private registry. docker login ``` -1. Tag and push the image. Replace `` with your registry's path and `` with the version you're using, for example `3.2.1`: +1. Tag and push the image. Replace `` with your registry's path and `` with the version you're using, for example `3.2.2`: - For NGINX Plus Ingress Controller, run: diff --git a/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md b/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md index 0efd40249c1..a3046812c6e 100644 --- a/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md +++ b/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md @@ -159,9 +159,9 @@ You can also use the certificate and key from the MyF5 portal and the Docker reg { "name": "nginx-ic/nginx-plus-ingress", "tags": [ - "3.2.1-alpine", - "3.2.1-ubi", - "3.2.1" + "3.2.2-alpine", + "3.2.2-ubi", + "3.2.2" ] } @@ -169,8 +169,8 @@ You can also use the certificate and key from the MyF5 portal and the Docker reg { "name": "nginx-ic-nap/nginx-plus-ingress", "tags": [ - "3.2.1-ubi", - "3.2.1" + "3.2.2-ubi", + "3.2.2" ] } @@ -178,8 +178,8 @@ You can also use the certificate and key from the MyF5 portal and the Docker reg { "name": "nginx-ic-dos/nginx-plus-ingress", "tags": [ - "3.2.1-ubi", - "3.2.1" + "3.2.2-ubi", + "3.2.2" ] } ``` diff --git a/docs/content/overview/controller-comparison.md b/docs/content/overview/controller-comparison.md index ff506c17fe2..7a726be8127 100644 --- a/docs/content/overview/controller-comparison.md +++ b/docs/content/overview/controller-comparison.md @@ -25,11 +25,11 @@ The table below summarizes the key difference between nginxinc/kubernetes-ingres | NGINX version | [Custom](https://github.com/kubernetes/ingress-nginx/tree/main/images/nginx) NGINX build that includes several third-party modules | NGINX official mainline [build](https://github.com/nginxinc/docker-nginx) | NGINX Plus | | Commercial support | N/A | N/A | Included | | **Load balancing configuration via the Ingress resource** | -| Merging Ingress rules with the same host | Supported | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/mergeable-ingress-types) | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/mergeable-ingress-types) | +| Merging Ingress rules with the same host | Supported | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/mergeable-ingress-types) | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/mergeable-ingress-types) | | HTTP load balancing extensions - Annotations | See the [supported annotations](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/) | See the [supported annotations](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/) | See the [supported annotations](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/)| | HTTP load balancing extensions -- ConfigMap | See the [supported ConfigMap keys](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/) | See the [supported ConfigMap keys](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) | See the [supported ConfigMap keys](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) | | TCP/UDP | Supported via a ConfigMap | Supported via custom resources | Supported via custom resources | -| Websocket | Supported | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/websocket) | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/websocket) | +| Websocket | Supported | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/websocket) | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/websocket) | | TCP SSL Passthrough | Supported via a ConfigMap | Supported via custom resources | Supported via custom resources | | JWT validation | Not supported | Not supported | Supported | | Session persistence | Supported via a third-party module | Not supported | Supported | diff --git a/docs/content/overview/nginx-plus.md b/docs/content/overview/nginx-plus.md index 47b06fb9015..bf3d3eef50e 100644 --- a/docs/content/overview/nginx-plus.md +++ b/docs/content/overview/nginx-plus.md @@ -16,9 +16,9 @@ The NGINX Ingress Controller works with [NGINX](https://nginx.org/) as well as [ - _Real-time metrics_: Metrics for NGINX Plus and application performance are available through the API or the [NGINX Status Page]({{< relref "logging-and-monitoring/status-page">}}). These metrics can also be exported to [Prometheus]({{< relref "logging-and-monitoring/prometheus">}}). - _Additional load balancing methods_: The `least_time` and `random two least_time` methods and their derivatives become available. The NGINX [`ngx_http_upstream_module` documentation](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) has the complete list of load balancing methods. -- _Session persistence_: The *sticky cookie* method becomes available. See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/session-persistence) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/session-persistence) examples. -- _Active health checks_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/health-checks) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/health-checks) examples. -- _JWT validation_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/ingress-resources/jwt) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.1/examples/custom-resources/jwt) examples. +- _Session persistence_: The *sticky cookie* method becomes available. See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/session-persistence) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/session-persistence) examples. +- _Active health checks_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/health-checks) examples. +- _JWT validation_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/jwt) examples. For a comprehensive guide of NGINX Plus features available with Ingress resources, see the [ConfigMap]({{< relref "configuration/global-configuration/configmap-resource">}}) and [Annotations]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations">}}) documentation. diff --git a/docs/content/releases.md b/docs/content/releases.md index b02292375fd..e5b32c7040a 100644 --- a/docs/content/releases.md +++ b/docs/content/releases.md @@ -6,6 +6,124 @@ doctypes: ["concept"] toc: true docs: "DOCS-616" --- +
+ +## 3.3.2 + +1 Nov 2023 + +### Fixes + +- [4578](https://github.com/nginxinc/kubernetes-ingress/pull/4578) Update Dockerfile to add user creation for NGINX Plus images. + +### Dependencies + +- [4572](https://github.com/nginxinc/kubernetes-ingress/pull/4572) Update NGINX version to 1.25.3. +- [4569](https://github.com/nginxinc/kubernetes-ingress/pull/4569), [4591](https://github.com/nginxinc/kubernetes-ingress/pull/4591) Bump Go dependencies. + +### Upgrade + +- For NGINX, use the 3.3.2 images from our +[DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.3.2), +[GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), +[Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). +- For NGINX Plus, use the 3.3.2 images from the F5 Container registry, +the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), +the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) +or build your own image using the 3.3.2 source code +- For Helm, use version 1.0.2 of the chart. + +
+ +## 3.3.1 + +13 Oct 2023 + +### Overview + +This releases updates NGINX Plus to R30 P1 and dependencies to mitigate HTTP/2 Rapid Reset Attack vulnerability [CVE-2023-44487](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2023-44487). + +### Dependencies + +- [4501](https://github.com/nginxinc/kubernetes-ingress/pull/4501) Bump Go to 1.21.3 +- [4502](https://github.com/nginxinc/kubernetes-ingress/pull/4502), [4514](https://github.com/nginxinc/kubernetes-ingress/pull/4514) Bump Go dependencies. + +### Upgrade + +- For NGINX, use the 3.3.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.3.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). +- For NGINX Plus, use the 3.3.1 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.3.1 source code + +
+ +## 3.3.0 + +26 Sep 2023 + +### Overview + +With release 3.3 the NGINX Ingress Controller continues to advance capabilities for an ever-demanding set of use cases +that go beyond simple layer 7 routing for services running exclusively in Kubernetes. + +When involved in diagnostic operations and viewing the NGINX Plus console or when viewing the enhanced NGINX Plus +metrics through Prometheus, customers now enjoy the added dimension of the backend service being available to aide in +identification of issues as well as observing performance. + +50% of our users continue to rely heavily on the Ingress resource and its "mergeable Ingress" usage pattern, to enhance +the experience for these customers we have added the path-regex annotation with support for case sensitive, case +insensitive, as well as exact regex match patterns. + +Prometheus continues to be the most popular metrics platform for Kubernetes users. To further enhance ease of setting up +integration with Prometheus we have finalized support for the Prometheus serviceMonitor capability. Providing better +scraping controls for Prometheus admins. + +For our most demanding customers performing a blue / green upgrade of the Ingress Controller itself supports the ability +to provide their business customers an enhanced experience with no loss of session fidelity. Support for this pattern +and others has been added through Helm chart enhancement that allows two deployments to share a single ingressClass +resource and duplicate the same configuration. + +To accommodate these enhancements, several new values have been added to our Helm chart, as well as modifications to +existing values. Due to the potential impacts of these changes we have issued a major release to the Helm chart, +advancing to v1.0.0 + +To better align with the demands of supporting additional protocols such as MQTT and QUIC, the NGINX Ingress Controller +is changing how listeners are defined for HTTP traffic. You have always had controls over the ports defined for +TCP/UDP traffic through the GlobalConfiguration and TransportServer objects. That same flexibility has been introduced +for HTTP/S traffic and the VirtualServer. This area will continue to expand to give customers full control over NGINX +listeners so they can tailor to their specific needs and policies. + +### Features + +- [4023](https://github.com/nginxinc/kubernetes-ingress/pull/4023) Read Prometheus key/cert from memory. +- [4080](https://github.com/nginxinc/kubernetes-ingress/pull/4080) Expose Location Zones metrics. +- [4127](https://github.com/nginxinc/kubernetes-ingress/pull/4127), [4200](https://github.com/nginxinc/kubernetes-ingress/pull/4200), [4223](https://github.com/nginxinc/kubernetes-ingress/pull/4223) Add path-regex annotation for ingress. +- [4108](https://github.com/nginxinc/kubernetes-ingress/pull/4108) Add command line argument for custom TLS Passthrough port. +- [4271](https://github.com/nginxinc/kubernetes-ingress/pull/4271) Add custom listener controls to VirtualServer. + +### Fixes + +- [4160](https://github.com/nginxinc/kubernetes-ingress/pull/4160) Update JWT/JWKS policy validation. +- [4371](https://github.com/nginxinc/kubernetes-ingress/pull/4371) Improve runtime batch reloads. + +### Helm Chart + +- [3977](https://github.com/nginxinc/kubernetes-ingress/pull/3977) Add support for controller.selectorLabels. Thanks to [Youqing Han](https://github.com/hanyouqing). +- [4058](https://github.com/nginxinc/kubernetes-ingress/pull/4058) Add clusterIP to service if specified in values. Thanks to [EutiziStefano](https://github.com/EutiziStefano). +- [4252](https://github.com/nginxinc/kubernetes-ingress/pull/4252) Make containerPort and hostPort customizable. +- [4331](https://github.com/nginxinc/kubernetes-ingress/pull/4331) Expose Prometheus metrics through a headless Service. +- [4351](https://github.com/nginxinc/kubernetes-ingress/pull/4351) Update helm values file to move controller.serviceMonitor to prometheus.serviceMonitor. +- [4333](https://github.com/nginxinc/kubernetes-ingress/pull/4333) Allow installing IC without creating a new ingress class. + +### Upgrade + +- For NGINX, use the 3.3.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.3.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). +- For NGINX Plus, use the 3.3.0 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.3.0 source code +- For Helm, use version 1.0.0 of the chart. + +### Supported Platforms + +We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.22-1.28. + +
## NGINX Ingress Controller 3.2.1 diff --git a/docs/content/technical-specifications.md b/docs/content/technical-specifications.md index b1ad5669c25..97033f22edd 100644 --- a/docs/content/technical-specifications.md +++ b/docs/content/technical-specifications.md @@ -23,7 +23,7 @@ We explicitly test NGINX Ingress Controller on a range of Kubernetes platforms f {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} | NIC Version | Supported Kubernetes Version | NIC Helm Chart Version | NIC Operator Version | NGINX / NGINX Plus version | | --- | --- | --- | --- | --- | -| 3.2.1 | 1.27 - 1.22 | 0.18.1 | 1.5.1 | 1.25.2 / R30 | +| 3.2.2 | 1.27 - 1.22 | 0.18.1 | 1.5.1 | 1.25.2 / R30 | | 3.1.1 | 1.26 - 1.22 | 0.17.1 | 1.4.2 | 1.23.4 / R29 | | 3.0.2 | 1.26 - 1.21 | 0.16.2 | 1.3.1 | 1.23.3 / R28 | | 2.4.2 | 1.25 - 1.19 | 0.15.2 | 1.2.1 | 1.23.2 / R28 | @@ -51,9 +51,9 @@ _All images include NGINX 1.25.2._ {{< bootstrap-table "table table-bordered table-responsive" >}} |
Name
|
Base image
|
Third-party modules
| DockerHub image | Architectures | | ---| --- | --- | --- | --- | -|Alpine-based image | ``nginx:1.25.2-alpine``,
based on on ``alpine:3.18`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.1-alpine`` | arm/v7
arm64
amd64
ppc64le
s390x | -|Debian-based image | ``nginx:1.25.2``,
based on on ``debian:12-slim`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.1`` | arm/v7
arm64
amd64
ppc64le
s390x | -|Ubi-based image | ``nginxcontrib/nginx:1.25.2-ubi``,
based on on ``redhat/ubi9-minimal`` | | ``nginx/nginx-ingress:3.2.1-ubi`` | arm64
amd64
ppc64le
s390x | +|Alpine-based image | ``nginx:1.25.2-alpine``,
based on on ``alpine:3.18`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.2-alpine`` | arm/v7
arm64
amd64
ppc64le
s390x | +|Debian-based image | ``nginx:1.25.2``,
based on on ``debian:12-slim`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.2`` | arm/v7
arm64
amd64
ppc64le
s390x | +|Ubi-based image | ``nginxcontrib/nginx:1.25.2-ubi``,
based on on ``redhat/ubi9-minimal`` | | ``nginx/nginx-ingress:3.2.2-ubi`` | arm64
amd64
ppc64le
s390x | {{% /bootstrap-table %}} ### Images with NGINX Plus @@ -67,16 +67,16 @@ NGINX Plus images are available through the F5 Container registry `private-regis {{< bootstrap-table "table table-striped table-bordered table-responsive" >}} |
Name
|
Base image
|
Third-party modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.1-alpine` | arm64
amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:3.2.1-alpine-fips` | arm64
amd64 | -|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.1` | arm64
amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:3.2.1` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-dos/nginx-plus-ingress:3.2.1` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.1` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:3.2.1-ubi` | arm64
amd64
s390x | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi8`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:3.2.1-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:3.2.1-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.1-ubi` | amd64 | +|Alpine-based image | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.2-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:3.2.2-alpine-fips` | arm64
amd64 | +|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.2` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:3.2.2` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-dos/nginx-plus-ingress:3.2.2` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.2` | amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:3.2.2-ubi` | arm64
amd64
s390x | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi8`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:3.2.2-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:3.2.2-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.2-ubi` | amd64 | {{% /bootstrap-table %}} #### **AWS Marketplace** diff --git a/docs/content/tutorials/oidc-custom-configuration.md b/docs/content/tutorials/oidc-custom-configuration.md index 2604cead1a7..6b9c2d6746b 100644 --- a/docs/content/tutorials/oidc-custom-configuration.md +++ b/docs/content/tutorials/oidc-custom-configuration.md @@ -30,7 +30,7 @@ Run the below command to generate a ConfigMap with the contents of the `oidc.con **NOTE** The ConfigMap must be deployed in the same `namespace` as the F5 NGINX Ingress Controller. ```console -kubectl create configmap oidc-config-map --from-literal=oidc.conf="$(curl -k https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.1/internal/configs/oidc/oidc.conf)" +kubectl create configmap oidc-config-map --from-literal=oidc.conf="$(curl -k https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/internal/configs/oidc/oidc.conf)" ``` Use the `kubectl describe` command to confirm the contents of the ConfigMap are correct. From 73ba6b60616c8f64a4fca8e24c52e30a016fc947 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 11:47:29 +0000 Subject: [PATCH 03/17] Fix version number --- .../configuration/configuration-examples.md | 4 +- .../configmap-resource.md | 16 ++++---- .../global-configuration/custom-templates.md | 2 +- .../handling-host-and-listener-collisions.md | 2 +- ...advanced-configuration-with-annotations.md | 38 +++++++++---------- .../ingress-resources/basic-configuration.md | 2 +- .../cross-namespace-configuration.md | 4 +- .../ingress-resources/custom-annotations.md | 4 +- docs/content/configuration/policy-resource.md | 4 +- .../configuration/transportserver-resource.md | 2 +- ...server-and-virtualserverroute-resources.md | 6 +-- .../building-nginx-ingress-controller.md | 4 +- .../installing-nic/installation-with-helm.md | 2 +- .../installation-with-manifests.md | 2 +- .../installation-with-operator.md | 2 +- .../app-protect-dos/configuration.md | 2 +- .../app-protect-dos/installation.md | 6 +-- .../app-protect-waf/configuration.md | 24 ++++++------ .../app-protect-waf/installation.md | 4 +- .../pulling-ingress-controller-image.md | 18 ++++----- .../using-the-jwt-token-docker-secret.md | 14 +++---- .../content/overview/controller-comparison.md | 4 +- docs/content/overview/nginx-plus.md | 6 +-- docs/content/technical-specifications.md | 28 +++++++------- .../tutorials/oidc-custom-configuration.md | 2 +- 25 files changed, 101 insertions(+), 101 deletions(-) diff --git a/docs/content/configuration/configuration-examples.md b/docs/content/configuration/configuration-examples.md index a4b30ac51eb..94af4ce2396 100644 --- a/docs/content/configuration/configuration-examples.md +++ b/docs/content/configuration/configuration-examples.md @@ -11,5 +11,5 @@ docs: "DOCS-584" Our [GitHub repo](https://github.com/nginxinc/kubernetes-ingress) includes a number of configuration examples: -- [*Examples of Custom Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. -- [*Examples of Ingress Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. +- [*Examples of Custom Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources) show how to advanced NGINX features by using VirtualServer, VirtualServerRoute, TransportServer and Policy Custom Resources. +- [*Examples of Ingress Resources*](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources) show how to use advanced NGINX features in Ingress resources with annotations. diff --git a/docs/content/configuration/global-configuration/configmap-resource.md b/docs/content/configuration/global-configuration/configmap-resource.md index c8050a04505..54772fbbbe1 100644 --- a/docs/content/configuration/global-configuration/configmap-resource.md +++ b/docs/content/configuration/global-configuration/configmap-resource.md @@ -91,10 +91,10 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``server-names-hash-max-size`` | Sets the value of the [server_names_hash_max_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#server_names_hash_max_size) directive. | ``1024`` | | |``map-hash-bucket-size`` | Sets the value of the [map_hash_bucket_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_bucket_size) directive.| ``256`` | | |``map-hash-max-size`` | Sets the value of the [map_hash_max_size](http://nginx.org/en/docs/http/ngx_http_map_module.html#map_hash_max_size) directive. | ``2048`` | | -|``resolver-addresses`` | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, ``kube-dns.kube-system.svc.cluster.local`` ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | -|``resolver-ipv6`` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | ``True`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | -|``resolver-valid`` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | -|``resolver-timeout`` | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | ``30s`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services). | +|``resolver-addresses`` | Sets the value of the [resolver](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver) addresses. Note: If you use a DNS name (for example, ``kube-dns.kube-system.svc.cluster.local`` ) as a resolver address, NGINX Plus will resolve it using the system resolver during the start and on every configuration reload. If the name cannot be resolved or the DNS server doesn't respond, NGINX Plus will fail to start or reload. To avoid this, we recommend using IP addresses as resolver addresses instead of DNS names. Supported in NGINX Plus only. | N/A | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | +|``resolver-ipv6`` | Enables IPv6 resolution in the resolver. Supported in NGINX Plus only. | ``True`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | +|``resolver-valid`` | Sets the time NGINX caches the resolved DNS records. Supported in NGINX Plus only. | TTL value of a DNS record | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | +|``resolver-timeout`` | Sets the [resolver_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#resolver_timeout) for name resolution. Supported in NGINX Plus only. | ``30s`` | [Support for Type ExternalName Services](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services). | |``keepalive-timeout`` | Sets the value of the [keepalive_timeout](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_timeout) directive. | ``65s`` | | |``keepalive-requests`` | Sets the value of the [keepalive_requests](https://nginx.org/en/docs/http/ngx_http_core_module.html#keepalive_requests) directive. | ``100`` | | |``variables-hash-bucket-size`` | Sets the value of the [variables_hash_bucket_size](https://nginx.org/en/docs/http/ngx_http_core_module.html#variables_hash_bucket_size) directive. | ``256`` | | @@ -109,9 +109,9 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``error-log-level`` | Sets the global [error log level](https://nginx.org/en/docs/ngx_core_module.html#error_log) for NGINX. | ``notice`` | | |``access-log-off`` | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log). | ``False`` | | |``default-server-access-log-off`` | Disables the [access log](https://nginx.org/en/docs/http/ngx_http_log_module.html#access_log) for the default server. If access log is disabled globally (``access-log-off: "True"``), then the default server access log is always disabled. | ``False`` | | -|``log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/shared-examples/custom-log-format). | +|``log-format`` | Sets the custom [log format](https://nginx.org/en/docs/http/ngx_http_log_module.html#log_format) for HTTP and HTTPS traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx.tmpl) for the access log. | [Custom Log Format](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/shared-examples/custom-log-format). | |``log-format-escaping`` | Sets the characters escaping for the variables of the log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | -|``stream-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx.tmpl). | | +|``stream-log-format`` | Sets the custom [log format](https://nginx.org/en/docs/stream/ngx_stream_log_module.html#log_format) for TCP, UDP, and TLS Passthrough traffic. For convenience, it is possible to define the log format across multiple lines (each line separated by ``\n``). In that case, the Ingress Controller will replace every ``\n`` character with a space character. All ``'`` characters must be escaped. | See the [template file](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx.tmpl). | | |``stream-log-format-escaping`` | Sets the characters escaping for the variables of the stream log format. Supported values: ``json`` (JSON escaping), ``default`` (the default escaping) ``none`` (disables escaping). | ``default`` | | {{% /table %}} @@ -147,7 +147,7 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |ConfigMap Key | Description | Default | Example | | ---| ---| ---| --- | |``http2`` | Enables HTTP/2 in servers with SSL enabled. | ``False`` | | -|``proxy-protocol`` | Enables PROXY Protocol for incoming connections. | ``False`` | [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/shared-examples/proxy-protocol). | +|``proxy-protocol`` | Enables PROXY Protocol for incoming connections. | ``False`` | [Proxy Protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/shared-examples/proxy-protocol). | {{% /table %}} ### Backend Services (Upstreams) @@ -171,7 +171,7 @@ See the doc about [VirtualServer and VirtualServerRoute resources](/nginx-ingres |``http-snippets`` | Sets a custom snippet in http context. | N/A | | |``location-snippets`` | Sets a custom snippet in location context. | N/A | | |``server-snippets`` | Sets a custom snippet in server context. | N/A | | -|``stream-snippets`` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/tcp-udp). | +|``stream-snippets`` | Sets a custom snippet in stream context. | N/A | [Support for TCP/UDP Load Balancing](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/tcp-udp). | |``main-template`` | Sets the main NGINX configuration template. | By default the template is read from the file in the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | |``ingress-template`` | Sets the NGINX configuration template for an Ingress resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | |``virtualserver-template`` | Sets the NGINX configuration template for an VirtualServer resource. | By default the template is read from the file on the container. | [Custom Templates](/nginx-ingress-controller/configuration/global-configuration/custom-templates). | diff --git a/docs/content/configuration/global-configuration/custom-templates.md b/docs/content/configuration/global-configuration/custom-templates.md index 6ff302a8eca..a3d1c51c8ec 100644 --- a/docs/content/configuration/global-configuration/custom-templates.md +++ b/docs/content/configuration/global-configuration/custom-templates.md @@ -9,4 +9,4 @@ docs: "DOCS-587" --- -The Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/shared-examples/custom-templates). +The Ingress Controller uses templates to generate NGINX configuration for Ingress resources, VirtualServer resources and the main NGINX configuration file. You can customize the templates and apply them via the ConfigMap. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/shared-examples/custom-templates). diff --git a/docs/content/configuration/handling-host-and-listener-collisions.md b/docs/content/configuration/handling-host-and-listener-collisions.md index 2975acd2d6b..04524891909 100644 --- a/docs/content/configuration/handling-host-and-listener-collisions.md +++ b/docs/content/configuration/handling-host-and-listener-collisions.md @@ -87,7 +87,7 @@ Similarly, if `cafe-ingress` was created first, it will win `cafe.example.com` a It is possible to merge configuration for multiple Ingress resources for the same host. One common use case for this approach is distributing resources across multiple namespaces. See the [Cross-namespace Configuration](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration/) doc for more information. -It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/cross-namespace-configuration) on GitHub. +It is *not* possible to merge the configurations for multiple VirtualServer resources for the same host. However, you can split the VirtualServers into multiple VirtualServerRoute resources, which a single VirtualServer can then reference. See the [corresponding example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/cross-namespace-configuration) on GitHub. It is *not* possible to merge configuration for multiple TransportServer resources. diff --git a/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md b/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md index 07d3c4cc545..41f5cb367c4 100644 --- a/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md +++ b/docs/content/configuration/ingress-resources/advanced-configuration-with-annotations.md @@ -122,7 +122,7 @@ The table below summarizes the available annotations. | ---| ---| ---| ---| --- | |``nginx.org/proxy-hide-headers`` | ``proxy-hide-headers`` | Sets the value of one or more [proxy_hide_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_hide_header) directives. Example: ``"nginx.org/proxy-hide-headers": "header-a,header-b"`` | N/A | | |``nginx.org/proxy-pass-headers`` | ``proxy-pass-headers`` | Sets the value of one or more [proxy_pass_header](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass_header) directives. Example: ``"nginx.org/proxy-pass-headers": "header-a,header-b"`` | N/A | | -|``nginx.org/rewrites`` | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [Rewrites Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/rewrites). | +|``nginx.org/rewrites`` | N/A | Configures URI rewriting using [proxy_pass](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_pass) directive. | N/A | [Rewrites Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/rewrites). | {{% /table %}} ### Auth and SSL/TLS @@ -138,10 +138,10 @@ The table below summarizes the available annotations. |``nginx.org/hsts-behind-proxy`` | ``hsts-behind-proxy`` | Enables HSTS based on the value of the ``http_x_forwarded_proto`` request header. Should only be used when TLS termination is configured in a load balancer (proxy) in front of the Ingress Controller. Note: to control redirection from HTTP to HTTPS configure the ``nginx.org/redirect-to-https`` annotation. | ``False`` | | |``nginx.org/basic-auth-secret`` | N/A | Specifies a Secret resource with a user list for HTTP Basic authentication. | N/A | | |``nginx.org/basic-auth-realm`` | N/A | Specifies a realm. | N/A | | -|``nginx.com/jwt-key`` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | -|``nginx.com/jwt-realm`` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | -|``nginx.com/jwt-token`` | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization`` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | -|``nginx.com/jwt-login-url`` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-key`` | N/A | Specifies a Secret resource with keys for validating JSON Web Tokens (JWTs). | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-realm`` | N/A | Specifies a realm. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-token`` | N/A | Specifies a variable that contains a JSON Web Token. | By default, a JWT is expected in the ``Authorization`` header as a Bearer Token. | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | +|``nginx.com/jwt-login-url`` | N/A | Specifies a URL to which a client is redirected in case of an invalid or missing JWT. | N/A | [Support for JSON Web Tokens (JWTs)](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt). | {{% /table %}} ### Listeners @@ -159,19 +159,19 @@ The table below summarizes the available annotations. |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | |``nginx.org/lb-method`` | ``lb-method`` | Sets the [load balancing method](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#choosing-a-load-balancing-method). To use the round-robin method, specify ``"round_robin"``. | ``"random two least_conn"`` | | -|``nginx.org/ssl-services`` | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [SSL Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/ssl-services). | -|``nginx.org/grpc-services`` | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2`` ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [GRPC Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/grpc-services). | -|``nginx.org/websocket-services`` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/websocket). | +|``nginx.org/ssl-services`` | N/A | Enables HTTPS or gRPC over SSL when connecting to the endpoints of services. | N/A | [SSL Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/ssl-services). | +|``nginx.org/grpc-services`` | N/A | Enables gRPC for services. Note: requires HTTP/2 (see ``http2`` ConfigMap key); only works for Ingresses with TLS termination enabled. | N/A | [GRPC Services Support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/grpc-services). | +|``nginx.org/websocket-services`` | N/A | Enables WebSocket for services. | N/A | [WebSocket support](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/websocket). | |``nginx.org/max-fails`` | ``max-fails`` | Sets the value of the [max_fails](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_fails) parameter of the ``server`` directive. | ``1`` | | |``nginx.org/max-conns`` | N\A | Sets the value of the [max_conns](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#max_conns) parameter of the ``server`` directive. | ``0`` | | |``nginx.org/upstream-zone-size`` | ``upstream-zone-size`` | Sets the size of the shared memory [zone](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#zone) for upstreams. For NGINX, the special value 0 disables the shared memory zones. For NGINX Plus, shared memory zones are required and cannot be disabled. The special value 0 will be ignored. | ``256K`` | | |``nginx.org/fail-timeout`` | ``fail-timeout`` | Sets the value of the [fail_timeout](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#fail_timeout) parameter of the ``server`` directive. | ``10s`` | | -|``nginx.com/sticky-cookie-services`` | N/A | Configures session persistence. | N/A | [Session Persistence](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/session-persistence). | +|``nginx.com/sticky-cookie-services`` | N/A | Configures session persistence. | N/A | [Session Persistence](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/session-persistence). | |``nginx.org/keepalive`` | ``keepalive`` | Sets the value of the [keepalive](https://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive) directive. Note that ``proxy_set_header Connection "";`` is added to the generated configuration when the value > 0. | ``0`` | | -|``nginx.com/health-checks`` | N/A | Enables active health checks. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | -|``nginx.com/health-checks-mandatory`` | N/A | Configures active health checks as mandatory. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | -|``nginx.com/health-checks-mandatory-queue`` | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | ``0`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | -|``nginx.com/slow-start`` | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks). | ``"0s"`` | | +|``nginx.com/health-checks`` | N/A | Enables active health checks. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | +|``nginx.com/health-checks-mandatory`` | N/A | Configures active health checks as mandatory. | ``False`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | +|``nginx.com/health-checks-mandatory-queue`` | N/A | When active health checks are mandatory, creates a queue where incoming requests are temporarily stored while NGINX Plus is checking the health of the endpoints after a configuration reload. | ``0`` | [Support for Active Health Checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | +|``nginx.com/slow-start`` | N/A | Sets the upstream server [slow-start period](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-load-balancer/#server-slow-start). By default, slow-start is activated after a server becomes [available](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#passive-health-checks) or [healthy](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#active-health-checks). To enable slow-start for newly-added servers, configure [mandatory active health checks](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks). | ``"0s"`` | | {{% /table %}} ### Snippets and Custom Templates @@ -190,11 +190,11 @@ The table below summarizes the available annotations. {{% table %}} |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | -|``appprotect.f5.com/app-protect-policy`` | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable`` is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-enable`` | N/A | Enable App Protect for the Ingress Resource. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log-enable`` | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log`` | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | -|``appprotect.f5.com/app-protect-security-log-destination`` | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | ``syslog:server=localhost:514`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-policy`` | N/A | The name of the App Protect Policy for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace of the Ingress Resource is used. If not specified but ``appprotect.f5.com/app-protect-enable`` is true, a default policy id applied. If the referenced policy resource does not exist, or policy is invalid, this annotation will be ignored, and the default policy will be applied. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-enable`` | N/A | Enable App Protect for the Ingress Resource. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log-enable`` | N/A | Enable the [security log](/nginx-app-protect/troubleshooting/#app-protect-logging-overview) for App Protect. | ``False`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log`` | N/A | The App Protect log configuration for the Ingress Resource. Format is ``namespace/name``. If no namespace is specified, the same namespace as the Ingress Resource is used. If not specified the default is used which is: filter: ``illegal``, format: ``default``. Multiple configurations can be specified in a comma separated list. Both log configurations and destinations list (see below) must be of equal length. Configs and destinations are paired by the list indices. | N/A | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | +|``appprotect.f5.com/app-protect-security-log-destination`` | N/A | The destination of the security log. For more information check the [DESTINATION argument](/nginx-app-protect/troubleshooting/#app-protect-logging-overview). Multiple destinations can be specified in a comma-separated list. Both log configurations and destinations list (see above) must be of equal length. Configs and destinations are paired by the list indices. | ``syslog:server=localhost:514`` | [Example for App Protect](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). | {{% /table %}} ### App Protect DoS @@ -204,5 +204,5 @@ The table below summarizes the available annotations. {{% table %}} |Annotation | ConfigMap Key | Description | Default | Example | | ---| ---| ---| ---| --- | -|``appprotectdos.f5.com/app-protect-dos-resource`` | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource](/nginx-ingress-controller/app-protect-dos/dos-protected/). | N/A | [Example for App Protect DoS](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-dos). | +|``appprotectdos.f5.com/app-protect-dos-resource`` | N/A | Enable App Protect DoS for the Ingress Resource by specifying a [DosProtectedResource](/nginx-ingress-controller/app-protect-dos/dos-protected/). | N/A | [Example for App Protect DoS](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-dos). | {{% /table %}} diff --git a/docs/content/configuration/ingress-resources/basic-configuration.md b/docs/content/configuration/ingress-resources/basic-configuration.md index f184609361a..742a60bbe10 100644 --- a/docs/content/configuration/ingress-resources/basic-configuration.md +++ b/docs/content/configuration/ingress-resources/basic-configuration.md @@ -53,7 +53,7 @@ Here is a breakdown of what this Ingress resource definition means: - The rule with the path `/coffee` instructs NGINX to distribute the requests with the `/coffee` URI among the pods of the *coffee* service, which is deployed with the name `coffee‑svc` in the cluster. - Both rules instruct NGINX to distribute the requests to `port 80` of the corresponding service (the `servicePort` field). -> For complete instructions on deploying the Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/complete-example) in our GitHub repository. +> For complete instructions on deploying the Ingress and Secret resources in the cluster, see the [complete example](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/complete-example) in our GitHub repository. > To learn more about the Ingress resource, see the [Ingress resource documentation](https://kubernetes.io/docs/concepts/services-networking/ingress/) in the Kubernetes docs. diff --git a/docs/content/configuration/ingress-resources/cross-namespace-configuration.md b/docs/content/configuration/ingress-resources/cross-namespace-configuration.md index 69aa78e51fc..ca0fbf090aa 100644 --- a/docs/content/configuration/ingress-resources/cross-namespace-configuration.md +++ b/docs/content/configuration/ingress-resources/cross-namespace-configuration.md @@ -9,6 +9,6 @@ docs: "DOCS-594" --- -You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. +You can spread the Ingress configuration for a common host across multiple Ingress resources using Mergeable Ingress resources. Such resources can belong to the *same* or *different* namespaces. This enables easier management when using a large number of paths. See the [Mergeable Ingress Resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/mergeable-ingress-types) example in our GitHub repo. -As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. +As an alternative to Mergeable Ingress resources, you can use [VirtualServer and VirtualServerRoute resources](/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/) for cross-namespace configuration. See the [Cross-Namespace Configuration](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/cross-namespace-configuration) example in our GitHub repo. diff --git a/docs/content/configuration/ingress-resources/custom-annotations.md b/docs/content/configuration/ingress-resources/custom-annotations.md index 68c927a0e66..b44da0b6d85 100644 --- a/docs/content/configuration/ingress-resources/custom-annotations.md +++ b/docs/content/configuration/ingress-resources/custom-annotations.md @@ -23,7 +23,7 @@ Custom annotations allow you to add an annotation for an NGINX feature that is n ## Usage -The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/internal/configs/version1/nginx-plus.ingress.tmpl). +The Ingress Controller generates NGINX configuration for Ingress resources by executing a configuration template. See [NGINX template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx.ingress.tmpl) or [NGINX Plus template](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/internal/configs/version1/nginx-plus.ingress.tmpl). To support custom annotations, the template has access to the information about the Ingress resource - its *name*, *namespace* and *annotations*. It is possible to check if a particular annotation present in the Ingress resource and conditionally insert NGINX configuration directives at multiple NGINX contexts - `http`, `server`, `location` or `upstream`. Additionally, you can get the value that is set to the annotation. @@ -139,4 +139,4 @@ deny all; ## Example -See the [custom annotations example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/ingress-resources/custom-annotations). +See the [custom annotations example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/ingress-resources/custom-annotations). diff --git a/docs/content/configuration/policy-resource.md b/docs/content/configuration/policy-resource.md index bc8c0017dfc..e24c9a01e52 100644 --- a/docs/content/configuration/policy-resource.md +++ b/docs/content/configuration/policy-resource.md @@ -12,7 +12,7 @@ The Policy resource allows you to configure features like access control and rat The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/access-control). +This document is the reference documentation for the Policy resource. An example of a Policy for access control is available in our [GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/access-control). ## Prerequisites @@ -459,7 +459,7 @@ NGINX Plus will pass the ID of an authenticated user to the backend in the HTTP #### Prerequisites In order to use OIDC, you need to enable [zone synchronization](https://docs.nginx.com/nginx/admin-guide/high-availability/zone_sync/). If you don't set up zone synchronization, NGINX Plus will fail to reload. -You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). +You also need to configure a resolver, which NGINX Plus will use to resolve the IDP authorization endpoint. You can find an example configuration [in our GitHub repository](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/oidc#step-7---configure-nginx-plus-zone-synchronization-and-resolver). > **Note**: The configuration in the example doesn't enable TLS and the synchronization between the replica happens in clear text. This could lead to the exposure of tokens. diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index ae0b0295c9f..e3967f0bb9c 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -10,7 +10,7 @@ docs: "DOCS-598" The TransportServer resource allows you to configure TCP, UDP, and TLS Passthrough load balancing. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the TransportServer resource. To see additional examples of using the resource for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources) folder in our GitHub repo. +This document is the reference documentation for the TransportServer resource. To see additional examples of using the resource for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources) folder in our GitHub repo. ## Prerequisites diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md index fc7ff34cb5c..c279ebc3ead 100644 --- a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md +++ b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md @@ -12,7 +12,7 @@ docs: "DOCS-599" The VirtualServer and VirtualServerRoute resources, introduced in release 1.5, enable use cases not supported with the Ingress resource, such as traffic splitting and advanced content-based routing. The resources are implemented as [Custom Resources](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -This document is the reference documentation for the resources. To see additional examples of using the resources for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources) folder in our GitHub repo. +This document is the reference documentation for the resources. To see additional examples of using the resources for specific use cases, go to the [examples/custom-resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources) folder in our GitHub repo. ## VirtualServer Specification @@ -346,7 +346,7 @@ tls: |Field | Description | Type | Required | | ---| ---| ---| --- | |``name`` | The name of the upstream. Must be a valid DNS label as defined in RFC 1035. For example, ``hello`` and ``upstream-123`` are valid. The name must be unique among all upstreams of the resource. | ``string`` | Yes | -|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | +|``service`` | The name of a [service](https://kubernetes.io/docs/concepts/services-networking/service/). The service must belong to the same namespace as the resource. If the service doesn't exist, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. For NGINX Plus only, services of type [ExternalName](https://kubernetes.io/docs/concepts/services-networking/service/#externalname) are also supported (check the [prerequisites](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/externalname-services#prerequisites) ). | ``string`` | Yes | |``subselector`` | Selects the pods within the service using label keys and values. By default, all pods of the service are selected. Note: the specified labels are expected to be present in the pods when they are created. If the pod labels are updated, the Ingress Controller will not see that change until the number of the pods is changed. | ``map[string]string`` | No | |``use-cluster-ip`` | Enables using the Cluster IP and port of the service instead of the default behavior of using the IP and port of the pods. When this field is enabled, the fields that configure NGINX behavior related to multiple upstream servers (like ``lb-method`` and ``next-upstream``) will have no effect, as the Ingress Controller will configure NGINX with only one upstream server that will match the service Cluster IP. | ``boolean`` | No | |``port`` | The port of the service. If the service doesn't define that port, NGINX will assume the service has zero endpoints and return a ``502`` response for requests for this upstream. The port must fall into the range ``1..65535``. | ``uint16`` | Yes | @@ -633,7 +633,7 @@ proxy: |``upstream`` | The name of the upstream which the requests will be proxied to. The upstream with that name must be defined in the resource. | ``string`` | Yes | |``requestHeaders`` | The request headers modifications. | [action.Proxy.RequestHeaders](#actionproxyrequestheaders) | No | |``responseHeaders`` | The response headers modifications. | [action.Proxy.ResponseHeaders](#actionproxyresponseheaders) | No | -|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/rewrites) example. | ``string`` | No | +|``rewritePath`` | The rewritten URI. If the route path is a regular expression -- starts with `~` -- the `rewritePath` can include capture groups with ``$1-9``. For example `$1` for the first group, and so on. For more information, check the [rewrite](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/rewrites) example. | ``string`` | No | {{% /table %}} ### Action.Proxy.RequestHeaders diff --git a/docs/content/installation/building-nginx-ingress-controller.md b/docs/content/installation/building-nginx-ingress-controller.md index 011d896e257..fc0cc16b3a0 100644 --- a/docs/content/installation/building-nginx-ingress-controller.md +++ b/docs/content/installation/building-nginx-ingress-controller.md @@ -41,10 +41,10 @@ Get your system ready for building and pushing the NGINX Ingress Controller imag cd kubernetes-ingress ``` - For instance if you want to clone version v3.2.2, the commands to run would be: + For instance if you want to clone version v3.3.2, the commands to run would be: ```shell - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.2 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 cd kubernetes-ingress ``` diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index bc486464249..9fd9a577892 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -309,7 +309,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **controller.logLevel** | The log level of the Ingress Controller. | 1 | | **controller.image.digest** | The image digest of the Ingress Controller. | None | | **controller.image.repository** | The image repository of the Ingress Controller. | nginx/nginx-ingress | -| **controller.image.tag** | The tag of the Ingress Controller image. | 3.2.2 | +| **controller.image.tag** | The tag of the Ingress Controller image. | 3.3.2 | | **controller.image.pullPolicy** | The pull policy for the Ingress Controller image. | IfNotPresent | | **controller.lifecycle** | The lifecycle of the Ingress Controller pods. | {} | | **controller.customConfigMap** | The name of the custom ConfigMap used by the Ingress Controller. If set, then the default config is ignored. | "" | diff --git a/docs/content/installation/installing-nic/installation-with-manifests.md b/docs/content/installation/installing-nic/installation-with-manifests.md index a9165071259..00066f15619 100644 --- a/docs/content/installation/installing-nic/installation-with-manifests.md +++ b/docs/content/installation/installing-nic/installation-with-manifests.md @@ -36,7 +36,7 @@ git clone https://github.com/nginxinc/kubernetes-ingress.git --branch Check out the complete [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-dos). +> Check out the complete [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-dos). ## App Protect DoS Configuration diff --git a/docs/content/installation/integrations/app-protect-dos/installation.md b/docs/content/installation/integrations/app-protect-dos/installation.md index 1b77dde38b7..61d60220d20 100644 --- a/docs/content/installation/integrations/app-protect-dos/installation.md +++ b/docs/content/installation/integrations/app-protect-dos/installation.md @@ -34,10 +34,10 @@ Get your system ready for building and pushing the NGINX Ingress Controller imag cd kubernetes-ingress ``` - For instance if you want to clone version v3.2.2, the commands to run would be: + For instance if you want to clone version v3.3.2, the commands to run would be: ```shell - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.2 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 cd kubernetes-ingress/deployments ``` @@ -177,7 +177,7 @@ To enable the NGINX App Protect DoS Module: {{< include "installation/manifests/verify-pods-are-running.md" >}} -For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-dos/configuration.md" >}}),the [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-dos). +For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-dos/configuration.md" >}}),the [NGINX Ingress Controller with App Protect DoS example for VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-dos) and the [NGINX Ingress Controller with App Protect DoS example for Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-dos). --- diff --git a/docs/content/installation/integrations/app-protect-waf/configuration.md b/docs/content/installation/integrations/app-protect-waf/configuration.md index 74b9d8af2d8..fb631d0153a 100644 --- a/docs/content/installation/integrations/app-protect-waf/configuration.md +++ b/docs/content/installation/integrations/app-protect-waf/configuration.md @@ -8,7 +8,7 @@ docs: "DOCS-578" aliases: ["/app-protect/configuration/"] --- -> Check out the complete NGINX Ingress Controller with NGINX App Protect WAF example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf). +> Check out the complete NGINX Ingress Controller with NGINX App Protect WAF example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf). ## Global Configuration @@ -245,8 +245,8 @@ These are the typical steps to deploy an OpenAPI protection Policy in NGINX Ingr 3. Make other custom changes if needed (e.g. enable Data Guard protection). 4. Use a tool to convert the result to YAML. There are many, for example: [`yq` utility](https://github.com/mikefarah/yq). 5. Add the YAML properties to create an `APPolicy` Custom Resource putting the policy itself (as in step 4) within the `spec` property of the Custom Resource. Refer to the [NGINX App Protect Policies](#nginx-app-protect-waf-policies) section above. -6. Create a `Policy` object which references the `APPolicy` Custom Resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/app-protect-waf/waf.yaml). -7. Finally, attach the `Policy` object to a `VirtualServer` resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.2.2/examples/custom-resources/app-protect-waf/virtual-server.yaml). +6. Create a `Policy` object which references the `APPolicy` Custom Resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/app-protect-waf/waf.yaml). +7. Finally, attach the `Policy` object to a `VirtualServer` resource as in [this example](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/examples/custom-resources/app-protect-waf/virtual-server.yaml). **Note**: You need to make sure that the server where the resource files are located is always available when you are compiling your policy. @@ -381,7 +381,7 @@ The `link` option is also available in the `openApiFileReference` property and i In this example we deploy NGINX Ingress Controller with NGINX Plus and NGINX App Protect WAF, deploy a simple web application, and then configure load balancing and WAF protection for that application using the VirtualServer resource. -**Note:** You can find the example, and the files referenced, on [GitHub](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-waf). +**Note:** You can find the example, and the files referenced, on [GitHub](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-waf). ## Prerequisites @@ -403,7 +403,7 @@ In this example we deploy NGINX Ingress Controller with NGINX Plus and NGINX App Create the application deployment and service: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/webapp.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/webapp.yaml ``` ### Step 2. Deploy the AP Policy @@ -411,15 +411,15 @@ Create the application deployment and service: 1. Create the syslog service and pod for the NGINX App Protect WAF security logs: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/syslog.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/syslog.yaml ``` 2. Create the User-Defined Signature, WAF policy, and log configuration: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/ap-apple-uds.yaml - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/ap-dataguard-alarm-policy.yaml - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/ap-logconf.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/ap-apple-uds.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/ap-dataguard-alarm-policy.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/ap-logconf.yaml ``` ### Step 3 - Deploy the WAF Policy @@ -427,7 +427,7 @@ Create the application deployment and service: Create the WAF policy ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/waf.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/waf.yaml ``` Note the NGINX App Protect WAF configuration settings in the Policy resource. They enable WAF protection by configuring NGINX App Protect WAF with the policy and log configuration created in the previous step. @@ -437,7 +437,7 @@ Create the WAF policy 1. Create the VirtualServer Resource: ```console - kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/virtual-server.yaml + kubectl apply -f https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/virtual-server.yaml ``` Note that the VirtualServer references the policy waf-policy created in Step 3. @@ -478,7 +478,7 @@ To access the application, curl the coffee and the tea services. We'll use the - ### Configuration Example of Virtual Server -Refer to GitHub repo for [Virtual Server example](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/examples/custom-resources/app-protect-waf/webapp.yaml). +Refer to GitHub repo for [Virtual Server example](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/custom-resources/app-protect-waf/webapp.yaml). ```yaml apiVersion: k8s.nginx.org/v1 diff --git a/docs/content/installation/integrations/app-protect-waf/installation.md b/docs/content/installation/integrations/app-protect-waf/installation.md index 18ab6b27523..696084e63dd 100644 --- a/docs/content/installation/integrations/app-protect-waf/installation.md +++ b/docs/content/installation/integrations/app-protect-waf/installation.md @@ -23,7 +23,7 @@ You can also [install the Ingress Controller with App Protect WAF by using Helm] 2. Clone the Ingress Controller repo: ```console - git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.2.2 + git clone https://github.com/nginxinc/kubernetes-ingress.git --branch v3.3.2 cd kubernetes-ingress ``` @@ -65,4 +65,4 @@ Take the steps below to set up and deploy the NGINX Ingress Controller and App P 3. Enable the App Protect WAF module by adding the `enable-app-protect` [cli argument]({{< relref "configuration/global-configuration/command-line-arguments.md#cmdoption-enable-app-protect" >}}) to your Deployment or DaemonSet file. 4. [Deploy the Ingress Controller]({{< relref "installation/installing-nic/installation-with-manifests.md#deploy-ingress-controller" >}}). -For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-waf/configuration.md" >}}) and the NGINX Ingress Controller with App Protect example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/app-protect-waf" >}}). +For more information, see the [Configuration guide]({{< relref "installation/integrations/app-protect-waf/configuration.md" >}}) and the NGINX Ingress Controller with App Protect example resources on GitHub [for VirtualServer resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/app-protect-waf) and [for Ingress resources](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/app-protect-waf" >}}). diff --git a/docs/content/installation/nic-images/pulling-ingress-controller-image.md b/docs/content/installation/nic-images/pulling-ingress-controller-image.md index d34fd1239d1..b0ab382c7ae 100644 --- a/docs/content/installation/nic-images/pulling-ingress-controller-image.md +++ b/docs/content/installation/nic-images/pulling-ingress-controller-image.md @@ -38,7 +38,7 @@ The steps provided are for Linux. For Mac or Windows, consult the [Docker for Ma Next, pull the image you need from `private-registry.nginx.com`. To find the correct image, consult the [Tech Specs guide]({{< relref "technical-specifications#images-with-nginx-plus" >}}). -To pull an image, follow these steps. Replace `` with the specific version you need, for example, `3.2.2`. +To pull an image, follow these steps. Replace `` with the specific version you need, for example, `3.3.2`. - For NGINX Plus Ingress Controller, run: @@ -65,9 +65,9 @@ $ curl https://private-registry.nginx.com/v2/nginx-ic/nginx-plus-ingress/tags/li { "name": "nginx-ic/nginx-plus-ingress", "tags": [ - "3.2.2-alpine", - "3.2.2-ubi", - "3.2.2" + "3.3.2-alpine", + "3.3.2-ubi", + "3.3.2" ] } @@ -75,8 +75,8 @@ $ curl https://private-registry.nginx.com/v2/nginx-ic-nap/nginx-plus-ingress/tag { "name": "nginx-ic-nap/nginx-plus-ingress", "tags": [ - "3.2.2-ubi", - "3.2.2" + "3.3.2-ubi", + "3.3.2" ] } @@ -84,8 +84,8 @@ $ curl https://private-registry.nginx.com/v2/nginx-ic-dos/nginx-plus-ingress/tag { "name": "nginx-ic-dos/nginx-plus-ingress", "tags": [ - "3.2.2-ubi", - "3.2.2" + "3.3.2-ubi", + "3.3.2" ] } ``` @@ -102,7 +102,7 @@ After pulling the image, tag it and upload it to your private registry. docker login ``` -1. Tag and push the image. Replace `` with your registry's path and `` with the version you're using, for example `3.2.2`: +1. Tag and push the image. Replace `` with your registry's path and `` with the version you're using, for example `3.3.2`: - For NGINX Plus Ingress Controller, run: diff --git a/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md b/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md index a3046812c6e..dbf1bcfd54b 100644 --- a/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md +++ b/docs/content/installation/nic-images/using-the-jwt-token-docker-secret.md @@ -159,9 +159,9 @@ You can also use the certificate and key from the MyF5 portal and the Docker reg { "name": "nginx-ic/nginx-plus-ingress", "tags": [ - "3.2.2-alpine", - "3.2.2-ubi", - "3.2.2" + "3.3.2-alpine", + "3.3.2-ubi", + "3.3.2" ] } @@ -169,8 +169,8 @@ You can also use the certificate and key from the MyF5 portal and the Docker reg { "name": "nginx-ic-nap/nginx-plus-ingress", "tags": [ - "3.2.2-ubi", - "3.2.2" + "3.3.2-ubi", + "3.3.2" ] } @@ -178,8 +178,8 @@ You can also use the certificate and key from the MyF5 portal and the Docker reg { "name": "nginx-ic-dos/nginx-plus-ingress", "tags": [ - "3.2.2-ubi", - "3.2.2" + "3.3.2-ubi", + "3.3.2" ] } ``` diff --git a/docs/content/overview/controller-comparison.md b/docs/content/overview/controller-comparison.md index 7a726be8127..008a731ee9f 100644 --- a/docs/content/overview/controller-comparison.md +++ b/docs/content/overview/controller-comparison.md @@ -25,11 +25,11 @@ The table below summarizes the key difference between nginxinc/kubernetes-ingres | NGINX version | [Custom](https://github.com/kubernetes/ingress-nginx/tree/main/images/nginx) NGINX build that includes several third-party modules | NGINX official mainline [build](https://github.com/nginxinc/docker-nginx) | NGINX Plus | | Commercial support | N/A | N/A | Included | | **Load balancing configuration via the Ingress resource** | -| Merging Ingress rules with the same host | Supported | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/mergeable-ingress-types) | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/mergeable-ingress-types) | +| Merging Ingress rules with the same host | Supported | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/mergeable-ingress-types) | Supported via [Mergeable Ingresses](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/mergeable-ingress-types) | | HTTP load balancing extensions - Annotations | See the [supported annotations](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/) | See the [supported annotations](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/) | See the [supported annotations](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/)| | HTTP load balancing extensions -- ConfigMap | See the [supported ConfigMap keys](https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/configmap/) | See the [supported ConfigMap keys](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) | See the [supported ConfigMap keys](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) | | TCP/UDP | Supported via a ConfigMap | Supported via custom resources | Supported via custom resources | -| Websocket | Supported | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/websocket) | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/websocket) | +| Websocket | Supported | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/websocket) | Supported via an [annotation](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/websocket) | | TCP SSL Passthrough | Supported via a ConfigMap | Supported via custom resources | Supported via custom resources | | JWT validation | Not supported | Not supported | Supported | | Session persistence | Supported via a third-party module | Not supported | Supported | diff --git a/docs/content/overview/nginx-plus.md b/docs/content/overview/nginx-plus.md index bf3d3eef50e..4123a889614 100644 --- a/docs/content/overview/nginx-plus.md +++ b/docs/content/overview/nginx-plus.md @@ -16,9 +16,9 @@ The NGINX Ingress Controller works with [NGINX](https://nginx.org/) as well as [ - _Real-time metrics_: Metrics for NGINX Plus and application performance are available through the API or the [NGINX Status Page]({{< relref "logging-and-monitoring/status-page">}}). These metrics can also be exported to [Prometheus]({{< relref "logging-and-monitoring/prometheus">}}). - _Additional load balancing methods_: The `least_time` and `random two least_time` methods and their derivatives become available. The NGINX [`ngx_http_upstream_module` documentation](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) has the complete list of load balancing methods. -- _Session persistence_: The *sticky cookie* method becomes available. See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/session-persistence) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/session-persistence) examples. -- _Active health checks_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/health-checks) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/health-checks) examples. -- _JWT validation_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/ingress-resources/jwt) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.2.2/examples/custom-resources/jwt) examples. +- _Session persistence_: The *sticky cookie* method becomes available. See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/session-persistence) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/session-persistence) examples. +- _Active health checks_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/health-checks) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/health-checks) examples. +- _JWT validation_: See the [Ingress Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/ingress-resources/jwt) and [Custom Resource](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/examples/custom-resources/jwt) examples. For a comprehensive guide of NGINX Plus features available with Ingress resources, see the [ConfigMap]({{< relref "configuration/global-configuration/configmap-resource">}}) and [Annotations]({{< relref "configuration/ingress-resources/advanced-configuration-with-annotations">}}) documentation. diff --git a/docs/content/technical-specifications.md b/docs/content/technical-specifications.md index 97033f22edd..187b809107f 100644 --- a/docs/content/technical-specifications.md +++ b/docs/content/technical-specifications.md @@ -23,7 +23,7 @@ We explicitly test NGINX Ingress Controller on a range of Kubernetes platforms f {{< bootstrap-table "table table-bordered table-striped table-responsive" >}} | NIC Version | Supported Kubernetes Version | NIC Helm Chart Version | NIC Operator Version | NGINX / NGINX Plus version | | --- | --- | --- | --- | --- | -| 3.2.2 | 1.27 - 1.22 | 0.18.1 | 1.5.1 | 1.25.2 / R30 | +| 3.3.2 | 1.27 - 1.22 | 0.18.1 | 1.5.1 | 1.25.2 / R30 | | 3.1.1 | 1.26 - 1.22 | 0.17.1 | 1.4.2 | 1.23.4 / R29 | | 3.0.2 | 1.26 - 1.21 | 0.16.2 | 1.3.1 | 1.23.3 / R28 | | 2.4.2 | 1.25 - 1.19 | 0.15.2 | 1.2.1 | 1.23.2 / R28 | @@ -51,9 +51,9 @@ _All images include NGINX 1.25.2._ {{< bootstrap-table "table table-bordered table-responsive" >}} |
Name
|
Base image
|
Third-party modules
| DockerHub image | Architectures | | ---| --- | --- | --- | --- | -|Alpine-based image | ``nginx:1.25.2-alpine``,
based on on ``alpine:3.18`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.2-alpine`` | arm/v7
arm64
amd64
ppc64le
s390x | -|Debian-based image | ``nginx:1.25.2``,
based on on ``debian:12-slim`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.2.2`` | arm/v7
arm64
amd64
ppc64le
s390x | -|Ubi-based image | ``nginxcontrib/nginx:1.25.2-ubi``,
based on on ``redhat/ubi9-minimal`` | | ``nginx/nginx-ingress:3.2.2-ubi`` | arm64
amd64
ppc64le
s390x | +|Alpine-based image | ``nginx:1.25.2-alpine``,
based on on ``alpine:3.18`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.3.2-alpine`` | arm/v7
arm64
amd64
ppc64le
s390x | +|Debian-based image | ``nginx:1.25.2``,
based on on ``debian:12-slim`` | NGINX OpenTracing module

OpenTracing library

OpenTracing tracers for Jaeger

Zipkin and Datadog | ``nginx/nginx-ingress:3.3.2`` | arm/v7
arm64
amd64
ppc64le
s390x | +|Ubi-based image | ``nginxcontrib/nginx:1.25.2-ubi``,
based on on ``redhat/ubi9-minimal`` | | ``nginx/nginx-ingress:3.3.2-ubi`` | arm64
amd64
ppc64le
s390x | {{% /bootstrap-table %}} ### Images with NGINX Plus @@ -67,16 +67,16 @@ NGINX Plus images are available through the F5 Container registry `private-regis {{< bootstrap-table "table table-striped table-bordered table-responsive" >}} |
Name
|
Base image
|
Third-party modules
| F5 Container Registry Image | Architectures | | ---| ---| --- | --- | --- | -|Alpine-based image | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.2-alpine` | arm64
amd64 | -|Alpine-based image with FIPS inside | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:3.2.2-alpine-fips` | arm64
amd64 | -|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.2.2` | arm64
amd64 | -|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:3.2.2` | amd64 | -|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-dos/nginx-plus-ingress:3.2.2` | amd64 | -|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.2` | amd64 | -|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:3.2.2-ubi` | arm64
amd64
s390x | -|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi8`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:3.2.2-ubi` | amd64 | -|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:3.2.2-ubi` | amd64 | -|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:3.2.2-ubi` | amd64 | +|Alpine-based image | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.3.2-alpine` | arm64
amd64 | +|Alpine-based image with FIPS inside | ``alpine:3.18`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog

FIPS module and OpenSSL configuration | `nginx-ic/nginx-plus-ingress:3.3.2-alpine-fips` | arm64
amd64 | +|Debian-based image | ``debian:12-slim`` | NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic/nginx-plus-ingress:3.3.2` | arm64
amd64 | +|Debian-based image with NGINX App Protect WAF | ``debian:11-slim`` | NGINX App Protect WAF

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap/nginx-plus-ingress:3.3.2` | amd64 | +|Debian-based image with NGINX App Protect DoS | ``debian:11-slim`` | NGINX App Protect DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-dos/nginx-plus-ingress:3.3.2` | amd64 | +|Debian-based image with NGINX App Protect WAF and DoS | ``debian:11-slim`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript and OpenTracing modules

OpenTracing tracers for Jaeger

Zipkin and Datadog | `nginx-ic-nap-dos/nginx-plus-ingress:3.3.2` | amd64 | +|Ubi-based image | ``redhat/ubi9-minimal`` | NGINX Plus JavaScript module | `nginx-ic/nginx-plus-ingress:3.3.2-ubi` | arm64
amd64
s390x | +|Ubi-based image with NGINX App Protect WAF | ``redhat/ubi8`` | NGINX App Protect WAF and NGINX Plus JavaScript module | `nginx-ic-nap/nginx-plus-ingress:3.3.2-ubi` | amd64 | +|Ubi-based image with NGINX App Protect DoS | ``redhat/ubi8`` | NGINX App Protect DoS and NGINX Plus JavaScript module | `nginx-ic-dos/nginx-plus-ingress:3.3.2-ubi` | amd64 | +|Ubi-based image with NGINX App Protect WAF and DoS | ``redhat/ubi8`` | NGINX App Protect WAF and DoS

NGINX Plus JavaScript module | `nginx-ic-nap-dos/nginx-plus-ingress:3.3.2-ubi` | amd64 | {{% /bootstrap-table %}} #### **AWS Marketplace** diff --git a/docs/content/tutorials/oidc-custom-configuration.md b/docs/content/tutorials/oidc-custom-configuration.md index 6b9c2d6746b..4f79bb0d8ec 100644 --- a/docs/content/tutorials/oidc-custom-configuration.md +++ b/docs/content/tutorials/oidc-custom-configuration.md @@ -30,7 +30,7 @@ Run the below command to generate a ConfigMap with the contents of the `oidc.con **NOTE** The ConfigMap must be deployed in the same `namespace` as the F5 NGINX Ingress Controller. ```console -kubectl create configmap oidc-config-map --from-literal=oidc.conf="$(curl -k https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.2.2/internal/configs/oidc/oidc.conf)" +kubectl create configmap oidc-config-map --from-literal=oidc.conf="$(curl -k https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/internal/configs/oidc/oidc.conf)" ``` Use the `kubectl describe` command to confirm the contents of the ConfigMap are correct. From a847da588983d0c837bd8f86154f2086a9bed819 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 12:25:20 +0000 Subject: [PATCH 04/17] First set of drift fixes --- .../command-line-arguments.md | 16 +++++++++++ .../globalconfiguration-resource.md | 27 +++++++++++++------ .../handling-host-and-listener-collisions.md | 6 ++--- .../ingress-resources/basic-configuration.md | 2 +- .../ingress-resources/custom-annotations.md | 11 +++++--- 5 files changed, 47 insertions(+), 15 deletions(-) diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index fc718ecfc0a..a1908311ed0 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -511,3 +511,19 @@ Disable IPV6 listeners explicitly for nodes that do not support the IPV6 stack. Default `false`. + +### -default-http-listener-port + +Sets the port for the HTTP `default_server` listener. + +Default `80`. + + + +### -default-https-listener-port + +Sets the port for the HTTPS `default_server` listener. + +Default `443`. + + \ No newline at end of file diff --git a/docs/content/configuration/global-configuration/globalconfiguration-resource.md b/docs/content/configuration/global-configuration/globalconfiguration-resource.md index e870fc06151..88578aa696b 100644 --- a/docs/content/configuration/global-configuration/globalconfiguration-resource.md +++ b/docs/content/configuration/global-configuration/globalconfiguration-resource.md @@ -11,7 +11,8 @@ docs: "DOCS-588" The GlobalConfiguration resource allows you to define the global configuration parameters of the Ingress Controller. The resource is implemented as a [Custom Resource](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/). -The resource supports configuring listeners for TCP and UDP load balancing. Listeners are required by [TransportServer resources](/nginx-ingress-controller/configuration/transportserver-resource). +The resource supports configuring listeners for TCP and UDP load balancing. Listeners are required by [TransportServer resources](/nginx-ingress-controller/configuration/transportserver-resource) and +can be used to configure custom listerners for VirtualServers as specified [here](/nginx-ingress-controller/tutorials/virtual-server-with-custom-listener-ports). ## Prerequisites @@ -22,7 +23,7 @@ When [installing](/nginx-ingress-controller/installation/installation-with-manif The GlobalConfiguration resource defines the global configuration parameters of the Ingress Controller. Below is an example: ```yaml -apiVersion: k8s.nginx.org/v1alpha1 +apiVersion: k8s.nginx.org/v1 kind: GlobalConfiguration metadata: name: nginx-configuration @@ -35,6 +36,13 @@ spec: - name: dns-tcp port: 5353 protocol: TCP + - name: http-8083 + port: 8083 + protocol: HTTP + - name: https-8443 + port: 8443 + protocol: HTTP + ssl: true ``` {{% table %}} @@ -45,12 +53,15 @@ spec: ### Listener -The listener defines a listener (a combination of a protocol and a port) that NGINX will use to accept traffic for a [TransportServer](/nginx-ingress-controller/configuration/transportserver-resource): +The `listeners:` key defines a listener (a combination of a protocol and a port) that NGINX will use to accept traffic for a [TransportServer](/nginx-ingress-controller/configuration/transportserver-resource) and a [VirtualServer](nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources): ```yaml -name: dns-tcp -port: 5353 -protocol: TCP +- name: dns-tcp + port: 5353 + protocol: TCP +- name: http-8083 + port: 8083 + protocol: HTTP ``` {{% table %}} @@ -100,7 +111,7 @@ If you try to create (or update) a resource that violates the structural schema ``` $ kubectl apply -f global-configuration.yaml - error: error validating "global-configuration.yaml": error validating data: ValidationError(GlobalConfiguration.spec.listeners[0].port): invalid type for org.nginx.k8s.v1alpha1.GlobalConfiguration.spec.listeners.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + error: error validating "global-configuration.yaml": error validating data: ValidationError(GlobalConfiguration.spec.listeners[0].port): invalid type for org.nginx.k8s.v1.GlobalConfiguration.spec.listeners.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false ``` - Example of Kubernetes API server validation: @@ -147,4 +158,4 @@ Events: Warning Rejected 6s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration is invalid and was rejected: spec.listeners: Duplicate value: "Duplicated port/protocol combination 53/UDP" ``` -Note how the events section includes a Warning event with the Rejected reason. +Note how the events section includes a Warning event with the Rejected reason. \ No newline at end of file diff --git a/docs/content/configuration/handling-host-and-listener-collisions.md b/docs/content/configuration/handling-host-and-listener-collisions.md index 04524891909..b4976d17e46 100644 --- a/docs/content/configuration/handling-host-and-listener-collisions.md +++ b/docs/content/configuration/handling-host-and-listener-collisions.md @@ -102,7 +102,7 @@ Consider the following two resources: - `tcp-1` TransportServer: ```yaml - apiVersion: k8s.nginx.org/v1alpha1 + apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: tcp-1 @@ -116,7 +116,7 @@ Consider the following two resources: - `tcp-2` TransportServer: ```yaml - apiVersion: k8s.nginx.org/v1alpha1 + apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: tcp-2 @@ -141,4 +141,4 @@ Events: Warning Rejected 10s nginx-ingress-controller Listener dns-tcp is taken by another resource ``` -Similarly, if `tcp-2` was created first, it will win `dns-tcp` and the Ingress Controller will reject `tcp-1`. +Similarly, if `tcp-2` was created first, it will win `dns-tcp` and the Ingress Controller will reject `tcp-1`. \ No newline at end of file diff --git a/docs/content/configuration/ingress-resources/basic-configuration.md b/docs/content/configuration/ingress-resources/basic-configuration.md index 742a60bbe10..9bf818470bd 100644 --- a/docs/content/configuration/ingress-resources/basic-configuration.md +++ b/docs/content/configuration/ingress-resources/basic-configuration.md @@ -113,7 +113,7 @@ The NGINX Ingress Controller imposes the following restrictions on Ingress resou - When defining an Ingress resource, the `host` field is required. - The `host` value needs to be unique among all Ingress and VirtualServer resources unless the Ingress resource is a [mergeable minion](/nginx-ingress-controller/configuration/ingress-resources/cross-namespace-configuration/). See also [Handling Host and Listener Collisions](/nginx-ingress-controller/configuration/handling-host-and-listener-collisions). - The `path` field in `spec.rules[].http.paths[]` is required for `Exact` and `Prefix` `pathTypes`. -- The ImplementationSpecific `pathType` is treated as equivilent to `Prefix` `pathType`, with the exception that when this `pathType` is configured, the `path` field in `spec.rules[].http.paths[]` is not mandatory. `path` defaults to `/` if not set but the `pathType` is set to ImplementationSpecific. +- The ImplementationSpecific `pathType` is treated as equivalent to `Prefix` `pathType`, with the exception that when this `pathType` is configured, the `path` field in `spec.rules[].http.paths[]` is not mandatory. `path` defaults to `/` if not set but the `pathType` is set to ImplementationSpecific. ## Advanced Configuration diff --git a/docs/content/configuration/ingress-resources/custom-annotations.md b/docs/content/configuration/ingress-resources/custom-annotations.md index b44da0b6d85..bd927d3b084 100644 --- a/docs/content/configuration/ingress-resources/custom-annotations.md +++ b/docs/content/configuration/ingress-resources/custom-annotations.md @@ -107,10 +107,15 @@ If you'd like to use custom annotations with Mergeable Ingress resources, please Helper functions can be used in the Ingress template to parse the values of custom annotations. {{% table %}} -|Function | Input Arguments | Return Arguments | Description | +| Function | Input Arguments | Return Arguments | Description | | ---| ---| ---| --- | -|``split`` | ``s, sep string`` | ``[]string`` | Splits the string ``s`` into a slice of strings separated by the ``sep``. | -|``trim`` | ``s string`` | ``string`` | Trims the trailing and leading whitespace from the string ``s``. | +| ``split`` | ``s, sep string`` | ``[]string`` | Splits the string ``s`` into a slice of strings separated by the ``sep``. | +| ``trim`` | ``s string`` | ``string`` | Trims the trailing and leading whitespace from the string ``s``. | +| ``contains`` | ``s, substr string`` | ``bool`` | Tests whether the string ``substr`` is a substring of the string ``s``. | +| ``hasPrefix`` | ``s, prefix string`` | ``bool`` | Tests whether the string ``prefix`` is a prefix of the string ``s``. | +| ``hasSuffix`` | ``s, suffix string`` | ``bool`` | Tests whether the string ``suffix`` is a suffix of the string ``s``. | +| ``toLower`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their lower case. | +| ``toUpper`` | ``s string`` | ``bool`` | Converts all letters in the string ``s`` to their upper case. | {{% /table %}} Consider the following custom annotation `custom.nginx.org/allowed-ips`, which expects a comma-separated list of IP addresses: From 2f50a35076dd1d8508d1c5b8712c0a6e354e94b7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:25:53 +0000 Subject: [PATCH 05/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../global-configuration/command-line-arguments.md | 2 +- .../global-configuration/globalconfiguration-resource.md | 2 +- .../configuration/handling-host-and-listener-collisions.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/content/configuration/global-configuration/command-line-arguments.md b/docs/content/configuration/global-configuration/command-line-arguments.md index a1908311ed0..b0239eca2bd 100644 --- a/docs/content/configuration/global-configuration/command-line-arguments.md +++ b/docs/content/configuration/global-configuration/command-line-arguments.md @@ -526,4 +526,4 @@ Sets the port for the HTTPS `default_server` listener. Default `443`. - \ No newline at end of file + diff --git a/docs/content/configuration/global-configuration/globalconfiguration-resource.md b/docs/content/configuration/global-configuration/globalconfiguration-resource.md index 88578aa696b..bfa29d6077a 100644 --- a/docs/content/configuration/global-configuration/globalconfiguration-resource.md +++ b/docs/content/configuration/global-configuration/globalconfiguration-resource.md @@ -158,4 +158,4 @@ Events: Warning Rejected 6s nginx-ingress-controller GlobalConfiguration nginx-ingress/nginx-configuration is invalid and was rejected: spec.listeners: Duplicate value: "Duplicated port/protocol combination 53/UDP" ``` -Note how the events section includes a Warning event with the Rejected reason. \ No newline at end of file +Note how the events section includes a Warning event with the Rejected reason. diff --git a/docs/content/configuration/handling-host-and-listener-collisions.md b/docs/content/configuration/handling-host-and-listener-collisions.md index b4976d17e46..8d176169137 100644 --- a/docs/content/configuration/handling-host-and-listener-collisions.md +++ b/docs/content/configuration/handling-host-and-listener-collisions.md @@ -141,4 +141,4 @@ Events: Warning Rejected 10s nginx-ingress-controller Listener dns-tcp is taken by another resource ``` -Similarly, if `tcp-2` was created first, it will win `dns-tcp` and the Ingress Controller will reject `tcp-1`. \ No newline at end of file +Similarly, if `tcp-2` was created first, it will win `dns-tcp` and the Ingress Controller will reject `tcp-1`. From 4791f2510405691d39c6ae971aac33392df29b60 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 12:38:05 +0000 Subject: [PATCH 06/17] Second set of drift fixes --- README.md | 4 ++-- .../configuration/transportserver-resource.md | 14 +++++++------- ...rtualserver-and-virtualserverroute-resources.md | 1 + .../content/troubleshooting/troubleshoot-common.md | 2 +- docs/content/tutorials/custom-listen-ports.md | 2 +- .../virtual-server-with-custom-listener-ports.md | 4 ++-- docs/content/usage-reporting.md | 2 +- 7 files changed, 15 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 49b64855878..3536f375a70 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ In the case of NGINX, the Ingress Controller is deployed in a pod along with the We publish NGINX Ingress Controller releases on GitHub. See our [releases page](https://github.com/nginxinc/kubernetes-ingress/releases). -The latest stable release is [3.3.1](https://github.com/nginxinc/kubernetes-ingress/releases/tag/v3.3.1). For production +The latest stable release is [3.3.2](https://github.com/nginxinc/kubernetes-ingress/releases/tag/v3.3.2). For production use, we recommend that you choose the latest stable release. The edge version is useful for experimenting with new features that are not yet published in a stable release. To use @@ -118,7 +118,7 @@ your links to the correct versions: | Version | Description | Image for NGINX | Image for NGINX Plus | Installation Manifests and Helm Chart | Documentation and Examples | | ------- | ----------- | --------------- | -------------------- | ---------------------------------------| -------------------------- | -| Latest stable release | For production use | Use the 3.3.1 images from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress) or [build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | Use the 3.3.1 images from the [F5 Container Registry](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or [Build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | [Manifests](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.1/deployments). [Helm chart](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.1/deployments/helm-chart). | [Documentation](https://docs.nginx.com/nginx-ingress-controller/). [Examples](https://docs.nginx.com/nginx-ingress-controller/configuration/configuration-examples/). | +| Latest stable release | For production use | Use the 3.3.2 images from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress) or [build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | Use the 3.3.2 images from the [F5 Container Registry](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or [Build your own image](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/). | [Manifests](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/deployments). [Helm chart](https://github.com/nginxinc/kubernetes-ingress/tree/v3.3.2/deployments/helm-chart). | [Documentation](https://docs.nginx.com/nginx-ingress-controller/). [Examples](https://docs.nginx.com/nginx-ingress-controller/configuration/configuration-examples/). | | Edge/Nightly | For testing and experimenting | Use the edge or nightly images from [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress) or [build your own image](https://github.com/nginxinc/kubernetes-ingress/tree/main/docs/content/installation/building-ingress-controller-image.md). | [Build your own image](https://github.com/nginxinc/kubernetes-ingress/tree/main/docs/content/installation/building-ingress-controller-image.md). | [Manifests](https://github.com/nginxinc/kubernetes-ingress/tree/main/deployments). [Helm chart](https://github.com/nginxinc/kubernetes-ingress/tree/main/charts/nginx-ingress). | [Documentation](https://github.com/nginxinc/kubernetes-ingress/tree/main/docs/content). [Examples](https://github.com/nginxinc/kubernetes-ingress/tree/main/examples). | ## SBOM (Software Bill of Materials) diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index e3967f0bb9c..79fe5bbe335 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -24,7 +24,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, - TCP load balancing: ```yaml - apiVersion: k8s.nginx.org/v1alpha1 + apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: dns-tcp @@ -45,7 +45,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, - UDP load balancing: ```yaml - apiVersion: k8s.nginx.org/v1alpha1 + apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: dns-udp @@ -67,7 +67,7 @@ The TransportServer resource defines load balancing configuration for TCP, UDP, - TLS passthrough load balancing: ```yaml - apiVersion: k8s.nginx.org/v1alpha1 + apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: secure-app @@ -299,7 +299,7 @@ In the kubectl get and similar commands, you can also use the short name `ts` in Snippets allow you to insert raw NGINX config into different contexts of NGINX configuration. In the example below, we use snippets to configure [access control](http://nginx.org/en/docs/stream/ngx_stream_access_module.html) in a TransportServer: ```yaml -apiVersion: k8s.nginx.org/v1alpha1 +apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: cafe @@ -317,7 +317,7 @@ spec: Snippets can also be specified for a stream. In the example below, we use snippets to [limit the number of connections](https://nginx.org/en/docs/stream/ngx_stream_limit_conn_module.html): ```yaml -apiVersion: k8s.nginx.org/v1alpha1 +apiVersion: k8s.nginx.org/v1 kind: TransportServer metadata: name: cafe @@ -365,7 +365,7 @@ If you try to create (or update) a resource that violates the structural schema ```console kubectl apply -f transport-server-passthrough.yaml - error: error validating "transport-server-passthrough.yaml": error validating data: ValidationError(TransportServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1alpha1.TransportServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false + error: error validating "transport-server-passthrough.yaml": error validating data: ValidationError(TransportServer.spec.upstreams[0].port): invalid type for org.nginx.k8s.v1.TransportServer.spec.upstreams.port: got "string", expected "integer"; if you choose to ignore these errors, turn validation off with --validate=false ``` - Example of Kubernetes API server validation: @@ -415,4 +415,4 @@ Note how the events section includes a Warning event with the Rejected reason. ## Customization via ConfigMap -The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. +The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. \ No newline at end of file diff --git a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md index c279ebc3ead..ad9eee5d76d 100644 --- a/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md +++ b/docs/content/configuration/virtualserver-and-virtualserverroute-resources.md @@ -127,6 +127,7 @@ cert-manager: |``duration`` | This field allows you to configure spec.duration field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | |``renew-before`` | this annotation allows you to configure spec.renewBefore field for the Certificate to be generated. Must be specified using a [Go time.Duration](https://pkg.go.dev/time#ParseDuration) string format, which does not allow the d (days) suffix. You must specify these values using s, m, and h suffixes instead. | ``string`` | No | |``usages`` | This field allows you to configure spec.usages field for the Certificate to be generated. Pass a string with comma-separated values i.e. ``key agreement,digital signature, server auth``. An exhaustive list of supported key usages can be found in the [the cert-manager api documentation](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.KeyUsage). | ``string`` | No | +|``issue-temp-cert`` | When ``true``, ask cert-manager for a [temporary self-signed certificate](https://cert-manager.io/docs/usage/certificate/#temporary-certificates-while-issuing) pending the issuance of the Certificate. This allows HTTPS-only servers to use ACME HTTP01 challenges when the TLS secret does not exist yet. | ``boolean`` | No | {{% /table %}} ### VirtualServer.Listener diff --git a/docs/content/troubleshooting/troubleshoot-common.md b/docs/content/troubleshooting/troubleshoot-common.md index fd704dd385b..63ff8f31e81 100644 --- a/docs/content/troubleshooting/troubleshoot-common.md +++ b/docs/content/troubleshooting/troubleshoot-common.md @@ -145,7 +145,7 @@ controller: nginxplus: plus image: repository: nginx/nginx-ingress - tag: 2.4.1 + tag: 3.3.0 # NGINX Configmap config: entries: diff --git a/docs/content/tutorials/custom-listen-ports.md b/docs/content/tutorials/custom-listen-ports.md index c11297f6f07..b7fcac13ca3 100644 --- a/docs/content/tutorials/custom-listen-ports.md +++ b/docs/content/tutorials/custom-listen-ports.md @@ -88,7 +88,7 @@ spec: spec: serviceAccountName: nginx-ingress containers: - - image: nginx/nginx-ingress:3.2.0 + - image: nginx/nginx-ingress:3.3.0 imagePullPolicy: IfNotPresent name: nginx-ingress ports: diff --git a/docs/content/tutorials/virtual-server-with-custom-listener-ports.md b/docs/content/tutorials/virtual-server-with-custom-listener-ports.md index 92dc50c9dd4..0bc62630f47 100644 --- a/docs/content/tutorials/virtual-server-with-custom-listener-ports.md +++ b/docs/content/tutorials/virtual-server-with-custom-listener-ports.md @@ -15,7 +15,7 @@ Each field must reference a valid listener defined by in a [GlobalConfiguration] 1. Create a yaml file called `nginx-configuration.yaml` with the below content: ```yaml -apiVersion: k8s.nginx.org/v1alpha1 +apiVersion: k8s.nginx.org/v1 kind: GlobalConfiguration metadata: name: nginx-configuration @@ -179,4 +179,4 @@ Server name: coffee-7dd75bc79b-qmhmv ... URI: /coffee ... -``` +``` \ No newline at end of file diff --git a/docs/content/usage-reporting.md b/docs/content/usage-reporting.md index 4a6d6f60232..6f900d812bd 100644 --- a/docs/content/usage-reporting.md +++ b/docs/content/usage-reporting.md @@ -87,7 +87,7 @@ To make the credential available to Usage Reporting, we need to create a Kuberne If you need to update the basic-auth credentials for NGINX Management Suite in the future, update the `username` and `password` fields, and apply the changes by running the command again. Usage Reporting will automatically detect the changes, using the new username and password without redeployment. -5. Download and save the deployment file [cluster-connector.yaml](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.1.1/examples/shared-examples/usage-reporting/cluster-connector.yaml). Edit the following under the `args` section and then save the file: +5. Download and save the deployment file [cluster-connector.yaml](https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v3.3.2/examples/shared-examples/usage-reporting/cluster-connector.yaml). Edit the following under the `args` section and then save the file: ```yaml args: From 4fe82ab5b1463cc1d6664aefef4c1db268690cc0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 12:38:32 +0000 Subject: [PATCH 07/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/content/configuration/transportserver-resource.md | 2 +- .../tutorials/virtual-server-with-custom-listener-ports.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/configuration/transportserver-resource.md b/docs/content/configuration/transportserver-resource.md index 79fe5bbe335..bfed15b4777 100644 --- a/docs/content/configuration/transportserver-resource.md +++ b/docs/content/configuration/transportserver-resource.md @@ -415,4 +415,4 @@ Note how the events section includes a Warning event with the Rejected reason. ## Customization via ConfigMap -The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. \ No newline at end of file +The [ConfigMap](/nginx-ingress-controller/configuration/global-configuration/configmap-resource) keys (except for `stream-snippets`, `stream-log-format`, `resolver-addresses`, `resolver-ipv6`, `resolver-valid` and `resolver-timeout`) do not affect TransportServer resources. diff --git a/docs/content/tutorials/virtual-server-with-custom-listener-ports.md b/docs/content/tutorials/virtual-server-with-custom-listener-ports.md index 0bc62630f47..a0badbe88fc 100644 --- a/docs/content/tutorials/virtual-server-with-custom-listener-ports.md +++ b/docs/content/tutorials/virtual-server-with-custom-listener-ports.md @@ -179,4 +179,4 @@ Server name: coffee-7dd75bc79b-qmhmv ... URI: /coffee ... -``` \ No newline at end of file +``` From 948e6c5fc9c317b4176731adfa48fcbeaae011a8 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 14:16:23 +0000 Subject: [PATCH 08/17] Change security configuration links to 3.2.2 tags instead of main --- docs/content/configuration/security.md | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/docs/content/configuration/security.md b/docs/content/configuration/security.md index 9097b580b70..2ec131f1872 100644 --- a/docs/content/configuration/security.md +++ b/docs/content/configuration/security.md @@ -22,13 +22,10 @@ In addition, the following relating more specifically to Ingress Controller. The Ingress Controller is deployed within a Kubernetes environment, this environment must be secured. Kubernetes uses [RBAC](https://kubernetes.io/docs/reference/access-authn-authz/rbac/) to control the resources and operations available to different types of users. The Ingress Controller requires a service account which is configured using RBAC. -We strongly recommend using the [RBAC configuration](https://github.com/nginxinc/kubernetes-ingress/blob/main/deployments/rbac/rbac.yaml) provided in our standard deployment configuration. -It is configured with the least amount of privilege required for the Ingress Controller to work. +We strongly recommend using the [RBAC configuration](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/deployments/rbac/rbac.yaml) provided in our standard deployment configuration. It is configured with the least amount of privilege required for the Ingress Controller to work. -We strongly recommend inspecting the RBAC configuration (for [manifests installation](https://github.com/nginxinc/kubernetes-ingress/blob/main/deployments/rbac/rbac.yaml) -or for [helm](https://github.com/nginxinc/kubernetes-ingress/blob/main/deployments/helm-chart/templates/rbac.yaml)) -to understand what access the Ingress Controller service account has and to which resources. -For example, by default the service account has access to all Secret resources in the cluster. +We strongly recommend inspecting the RBAC configuration for [Manifests](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/deployments/rbac/rbac.yaml) +or for [Helm](https://github.com/nginxinc/kubernetes-ingress/blob/v3.3.2/deployments/helm-chart/templates/rbac.yaml) to understand what access the Ingress Controller service account has and to which resources. For example, by default the service account has access to all Secret resources in the cluster. ### Certificates and Privacy Keys From 64c2bdcceadda7ac3b4b804284ed1bc64ea41b94 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 14:30:55 +0000 Subject: [PATCH 09/17] Re-hide comparison document This isn't up to date, and thus should not be published. --- docs/content/overview/controller-comparison.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/content/overview/controller-comparison.md b/docs/content/overview/controller-comparison.md index 008a731ee9f..df864a88cb5 100644 --- a/docs/content/overview/controller-comparison.md +++ b/docs/content/overview/controller-comparison.md @@ -5,6 +5,7 @@ weight: 400 doctypes: ["concept"] toc: true docs: "DOCS-610" +draft: true --- There are two NGINX-based Ingress Controller implementations out there: the one made by NGINX ([nginxinc/kubernetes-ingress](https://github.com/nginxinc/kubernetes-ingress)) and the one made by Kubernetes ([kubernetes/ingress-nginx](https://github.com/kubernetes/ingress-nginx)). In this document, we explain the key differences between those implementations. This information should help you to choose an appropriate implementation for your requirements or move from one implementation to the other. From be4ce73c7ccbaa354d82148be41915ad71b2e774 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 14:39:14 +0000 Subject: [PATCH 10/17] Remove hidden link --- docs/content/installation/ingress-nginx.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/content/installation/ingress-nginx.md b/docs/content/installation/ingress-nginx.md index deac49b786d..8501f2b82f8 100644 --- a/docs/content/installation/ingress-nginx.md +++ b/docs/content/installation/ingress-nginx.md @@ -19,8 +19,6 @@ authors: ["Jason Williams"] This page explains two different ways to migrate from the community-maintained [Ingress-NGINX Controller](https://github.com/kubernetes/ingress-nginx) project to NGINX Ingress Controller: using NGINX's Ingress Resources or with Kubernetes's built-in Ingress Resources. This is typically because of implementation differences, and to take advantage of features such as [NGINX Plus integration]({{}}). - - The information in this guide is extracted from a free eBook called "_Kubernetes Ingress Controller Deployment and Security with NGINX_", which can be downloaded from the [NGINX Library](https://www.nginx.com/resources/library/kubernetes-ingress-controller-deployment-security-nginx/). ## Before you begin From 932aa192f24a51d9d17a249e2f0e0cd200203178 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 15:11:08 +0000 Subject: [PATCH 11/17] Fix ordering of installation methods --- .../installation/installing-nic/installation-with-helm.md | 2 +- .../installation/installing-nic/installation-with-manifests.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index 9fd9a577892..8fd0d30b1f5 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -1,7 +1,7 @@ --- title: Installation with Helm description: This document explains how to install NGINX Ingress Controller in your Kubernetes cluster using Helm. -weight: 200 +weight: 100 doctypes: [""] toc: true docs: "DOCS-602" diff --git a/docs/content/installation/installing-nic/installation-with-manifests.md b/docs/content/installation/installing-nic/installation-with-manifests.md index 00066f15619..5047611e994 100644 --- a/docs/content/installation/installing-nic/installation-with-manifests.md +++ b/docs/content/installation/installing-nic/installation-with-manifests.md @@ -1,7 +1,7 @@ --- title: Installation with Manifests description: "This guide explains how to install NGINX Ingress Controller in a Kubernetes cluster using manifests. In addition, it provides instructions on how to set up role-based access control, create both common and custom resources, and uninstall NGINX Ingress Controller." -weight: 100 +weight: 200 doctypes: [""] aliases: - /installation/ From 43b3ed77d33f1e7458ca39a745129a04a0fa5c66 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Thu, 9 Nov 2023 15:20:58 +0000 Subject: [PATCH 12/17] Update theme to fix button issue --- docs/go.mod | 2 +- docs/go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/go.mod b/docs/go.mod index 587221b01bb..c00ee80586d 100644 --- a/docs/go.mod +++ b/docs/go.mod @@ -2,4 +2,4 @@ module github.com/nginxinc/kubernetes-ingress/docs go 1.19 -require github.com/nginxinc/nginx-hugo-theme v0.38.1 // indirect +require github.com/nginxinc/nginx-hugo-theme v0.40.0 // indirect diff --git a/docs/go.sum b/docs/go.sum index 1123743be06..b62d2cb4d4b 100644 --- a/docs/go.sum +++ b/docs/go.sum @@ -1,2 +1,4 @@ github.com/nginxinc/nginx-hugo-theme v0.38.1 h1:1dCP7tzeJTy0HOekaMn6glSAoCtD9bk3q9Xs4ZmPynU= github.com/nginxinc/nginx-hugo-theme v0.38.1/go.mod h1:DPNgSS5QYxkjH/BfH4uPDiTfODqWJ50NKZdorguom8M= +github.com/nginxinc/nginx-hugo-theme v0.40.0 h1:YP0I0+bRKcJ5WEb1s/OWcnlcvNvIcKscagJkCzsa+Vs= +github.com/nginxinc/nginx-hugo-theme v0.40.0/go.mod h1:DPNgSS5QYxkjH/BfH4uPDiTfODqWJ50NKZdorguom8M= From 68605a6d55de1967f000f6c49a9b00fd27fae4b8 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Mon, 13 Nov 2023 15:02:30 +0000 Subject: [PATCH 13/17] Update docs/content/installation/installing-nic/installation-with-manifests.md Co-authored-by: Shaun Signed-off-by: Alan Dooley --- .../installation/installing-nic/installation-with-manifests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/installation/installing-nic/installation-with-manifests.md b/docs/content/installation/installing-nic/installation-with-manifests.md index 5047611e994..37b4b38afb4 100644 --- a/docs/content/installation/installing-nic/installation-with-manifests.md +++ b/docs/content/installation/installing-nic/installation-with-manifests.md @@ -156,7 +156,7 @@ For more information about the _LoadBalancer_ service, refer to the [Kubernetes kubectl apply -f service/loadbalancer-aws-elb.yaml ``` - If yu're using AWS, Kubernetes will set up a Classic Load Balancer (ELB) in TCP mode. This load balancer will have the PROXY protocol enabled to pass along the client's IP address and port. + If you're using AWS, Kubernetes will set up a Classic Load Balancer (ELB) in TCP mode. This load balancer will have the PROXY protocol enabled to pass along the client's IP address and port. 2. AWS users: Follow these additional steps to work with ELB in TCP mode. From 936b7691b82ab4495d8e141106e50d72984be6ab Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Mon, 13 Nov 2023 15:16:00 +0000 Subject: [PATCH 14/17] Synchronise release note changes from 3.3 release branch --- docs/content/releases.md | 779 +++++++++++++++++---------------------- 1 file changed, 345 insertions(+), 434 deletions(-) diff --git a/docs/content/releases.md b/docs/content/releases.md index e5b32c7040a..230355743bc 100644 --- a/docs/content/releases.md +++ b/docs/content/releases.md @@ -1,7 +1,7 @@ --- title: Releases description: "NGINX Ingress Controller Release Notes." -weight: 300 +weight: 2100 doctypes: ["concept"] toc: true docs: "DOCS-616" @@ -125,31 +125,29 @@ We will provide technical support for NGINX Ingress Controller on any Kubernetes
-## NGINX Ingress Controller 3.2.1 +## 3.2.1 17 Aug 2023 -CHANGES: +### Dependencies - Update NGINX version to 1.25.2. - Update NGINX Plus version to R30. - Update Go to 1.21 and Go dependencies. -HELM CHART: - -- The version of the Helm chart is now 0.18.1. - -UPGRADE: +### Upgrade - For NGINX, use the 3.2.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.2.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.2.1 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.2.1 source code - For Helm, use version 0.18.1 of the chart. -## NGINX Ingress Controller 3.2.0 +
+ +## 3.2.0 27 June 2023 -FEATURES: +### Features - [3790](https://github.com/nginxinc/kubernetes-ingress/pull/3790) Gunzip for VS - [3863](https://github.com/nginxinc/kubernetes-ingress/pull/3863) OIDC - relaxed OIDC scope validation @@ -160,7 +158,7 @@ FEATURES: - [4022](https://github.com/nginxinc/kubernetes-ingress/pull/4022) Add document to tutorial section for configuring the default OIDC implementation. - [4031](https://github.com/nginxinc/kubernetes-ingress/pull/4031) Add NGINX Plus Alpine image with FIPS inside for supported (paid) customers. -FIXES: +### Fixes - [3737](https://github.com/nginxinc/kubernetes-ingress/pull/3737) Update VirtualServer to ignore CRL for EgressMTLS. - [3798](https://github.com/nginxinc/kubernetes-ingress/pull/3798) Update VirtualServer template to generate an internal jwt auth location per policy applied. @@ -168,32 +166,35 @@ FIXES: - [3870](https://github.com/nginxinc/kubernetes-ingress/pull/3870) Add Funcs() method to UpdateVirtualServerTemplate method. Thanks to [Bryan Hendryx](https://github.com/coolbry95). - [3933](https://github.com/nginxinc/kubernetes-ingress/pull/3933) fix --external-service flag when using serviceNameOverride. Thanks to [Tim N](https://github.com/timnee). -CHANGES: +### Dependencies - Update NGINX version to 1.25.1. - Update Debian to 12 for NGINX Plus images (except for images containing the NGINX App Protect modules). - Update Alpine to 3.18 for NGINX Plus images. -HELM CHART: +### Helm Chart - [3814](https://github.com/nginxinc/kubernetes-ingress/pull/3814) Remove semverCompare for allocateLoadBalancerNodePorts. Thanks to [Alex Wied](https://github.com/centromere). - [3905](https://github.com/nginxinc/kubernetes-ingress/pull/3905) Reverse order of NAPDOS maxDaemons and maxWorkers in Helm chart. -UPGRADE: +### Upgrade - For NGINX, use the 3.2.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.2.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.2.0 images from the F5 Container registry, the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE), the [GCP Marketplace](https://console.cloud.google.com/marketplace/browse?filter=partner:F5,%20Inc.&filter=solution-type:k8s&filter=category:networking) or build your own image using the 3.2.0 source code. - For Helm, use version 0.18.0 of the chart. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.22-1.27. -## NGINX Ingress Controller 3.1.1 +
+ +## 3.1.1 04 May 2023 -OVERVIEW: +### Overview + This release reverts the changes made in 3.1.0 to use sysctls to bind to lower level ports without the NET_BIND_SERVICE capability. It also adds support for serviceNameOverride in the Helm chart, that can be used to override the service name for the NGINX Ingress Controller. This is useful especially during an upgrade from versions prior to 3.1.0, to avoid downtime due to the service name change. To use this feature, set the `serviceNameOverride` value in the Helm chart to the name of the existing service. For example, if the existing service name is `my-release-nginx-ingress`, you can use `--set serviceNameOverride=my-release-nginx-ingress` when running the upgrade command. @@ -203,22 +204,22 @@ Here is an example upgrade command that keeps the existing service name `my-rele helm upgrade my-release oci://ghcr.io/nginxinc/charts/nginx-ingress --version 0.17.1 --set serviceNameOverride=my-release-nginx-ingress ``` -FIXES: +### Fixes - [3737](https://github.com/nginxinc/kubernetes-ingress/pull/3737) Update VirtualServer to ignore CRL for EgressMTLS. - [3722](https://github.com/nginxinc/kubernetes-ingress/pull/3722) Inherit NET_BIND_SERVICE from IC to Nginx. Thanks to [Valters Jansons](https://github.com/sigv). - [3798](https://github.com/nginxinc/kubernetes-ingress/pull/3798) Update VirtualServer template to generate an internal jwt auth location per policy applied. -IMPROVEMENTS: +### Features - [3491](https://github.com/nginxinc/kubernetes-ingress/pull/3491) Egress via Ingress VirtualServer Resource. -CHANGES: +### Dependencies - Update NGINX version to 1.23.4. - Update NGINX Plus version to R29. -HELM CHART: +### Helm Chart - [3602](https://github.com/nginxinc/kubernetes-ingress/pull/3602) Updated NGINX Service Mesh references in Helm templates. Thanks to [Jared Byers](https://github.com/jbyers19). - [3773](https://github.com/nginxinc/kubernetes-ingress/pull/3773) Swap cpu and memory in HPA template. Thanks to [Bryan Hendryx](https://github.com/coolbry95). @@ -226,17 +227,19 @@ HELM CHART: - [3815](https://github.com/nginxinc/kubernetes-ingress/pull/3815) Fix GlobalConfiguration name in Helm Chart. - [3862](https://github.com/nginxinc/kubernetes-ingress/pull/3862) Add correct indentation to controller-leader-election configmap helm template. -UPGRADE: +### Upgrade - For NGINX, use the 3.1.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.1.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.1.1 images from the F5 Container registry or build your own image using the 3.1.1 source code. - For Helm, use version 0.17.1 of the chart. -## NGINX Ingress Controller 3.1.0 +
+ +## 3.1.0 29 Mar 2023 -OVERVIEW: +### Overview - Beginning with release 3.1.0 the NET_BIND_SERVICE capability is no longer used, and instead relies on net.ipv4.ip_unprivileged_port_start sysctl to allow port binding. Kubernetes 1.22 or later is required for this sysctl to be [classified as safe](https://kubernetes.io/docs/tasks/administer-cluster/sysctl-cluster/#safe-and-unsafe-sysctls). **Ensure that you are using the latest updated `deployment` and `daemonset` example yaml files available in the repo.** - *The minimum supported version of Kubernetes is now 1.22*. The NGINX Ingress Controller now uses `sysctls` to [bind to lower level ports without additional privileges](https://github.com/nginxinc/kubernetes-ingress/pull/3573/). This removes the need to use `NET_BIND_SERVICE` to bind to these ports. Thanks to [Valters Jansons](https://github.com/sigv) for making this feature possible! @@ -248,7 +251,7 @@ OVERVIEW: - The NGINX Ingress Controller uses the latest OIDC reference implementation which now supports [forwarding access tokens to upstreams / backends](https://github.com/nginxinc/kubernetes-ingress/pull/3474). Thanks to [Shawn Kim](https://github.com/shawnhankim) for making this possible! - The default TLS secret is now optional. This improves the security posture of NGINX Ingress Controller through enabling [ssl_reject_handshake](http://nginx.org/en/docs/http/ngx_http_ssl_module.html#ssl_reject_handshake). This has the impact of immediately terminating the SSL handshake and not revealing TLS or cypher settings to calls that do not match a configured hostname. -FEATURES: +### Features - [3034](https://github.com/nginxinc/kubernetes-ingress/pull/3034) Allow extra args to be provided to the OIDC auth endpoint. Thanks to [Alan Wilkie](https://github.com/alanwilkie-finocomp). - [3474](https://github.com/nginxinc/kubernetes-ingress/pull/3474) Add access token support in the OIDC. Thanks to [Shawn Kim](https://github.com/shawnhankim). @@ -259,20 +262,17 @@ FEATURES: - [3451](https://github.com/nginxinc/kubernetes-ingress/pull/3451) Enable keepalive-time for healthchecks in VS and VSR. - [3560](https://github.com/nginxinc/kubernetes-ingress/pull/3560) Add support for load a pre-compiles AppProtect Policy Bundle. - [3632](https://github.com/nginxinc/kubernetes-ingress/pull/3632) Update nginx.org/ca secret type & crl field to IngressMTLS to support CRL. - -IMPROVEMENTS: - - [3629](https://github.com/nginxinc/kubernetes-ingress/pull/3629) Use the "runtime default" seccomp profile. Thanks to [Valters Jansons](https://github.com/sigv). - [3573](https://github.com/nginxinc/kubernetes-ingress/pull/3573) Rework port binding logic without privileges. Thanks to [Valters Jansons](https://github.com/sigv). - [3646](https://github.com/nginxinc/kubernetes-ingress/pull/3646) Remove app protect agent. - [3507](https://github.com/nginxinc/kubernetes-ingress/pull/3507) Support empty path for ImplementationSpecific pathType. - [3482](https://github.com/nginxinc/kubernetes-ingress/pull/3482) Use new NSM Spiffe and Cert rotation library. - [3442](https://github.com/nginxinc/kubernetes-ingress/pull/3442) Add websocket protocol option to monitor directive. -- [3674](https://github.com/nginxinc/kubernetes-ingress/pull/3674) Move NAP DoS chart to new repo +- [3674](https://github.com/nginxinc/kubernetes-ingress/pull/3674) Move NAP DoS chart to new repo. - [3302](https://github.com/nginxinc/kubernetes-ingress/pull/3302) Make default-server-secret optional. -- [3586](https://github.com/nginxinc/kubernetes-ingress/pull/3586) Add new labels and metadata to add version information to pods +- [3586](https://github.com/nginxinc/kubernetes-ingress/pull/3586) Add new labels and metadata to add version information to pods. -FIXES: +### Fixes - [3463](https://github.com/nginxinc/kubernetes-ingress/pull/3463) Support non-vs created Challenge Ingress. - [3475](https://github.com/nginxinc/kubernetes-ingress/pull/3475) Ensure leader election is correctly disabled when option is set to `false` in helm template. @@ -282,7 +282,7 @@ FIXES: - [3607](https://github.com/nginxinc/kubernetes-ingress/pull/3607) Clear Content-Length headers for requests processed by internal JWKS routes. - [3660](https://github.com/nginxinc/kubernetes-ingress/pull/3660) Remove unwanted chars from label value. -HELM CHART: +### Helm Chart - [3581](https://github.com/nginxinc/kubernetes-ingress/pull/3581) Push edge Helm Chart to OCI registries. - [3449](https://github.com/nginxinc/kubernetes-ingress/pull/3449) Correct values.schema.json nodeSelector. @@ -291,63 +291,61 @@ HELM CHART: - [3537](https://github.com/nginxinc/kubernetes-ingress/pull/3537) Update schema references to k8s v1.26.1. - [3606](https://github.com/nginxinc/kubernetes-ingress/pull/3606) Fix Helm Chart labels and templates. Move version update to labels. -UPGRADE: +### Upgrade - Make sure the Kubernetes version is in the supported platforms listed below. - For NGINX, use the 3.1.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.1.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.1.0 images from the F5 Container registry or build your own image using the 3.1.0 source code. - For Helm, use version 0.17.0 of the chart. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.22-1.26. -## NGINX Ingress Controller 3.0.2 +
+ +## 3.0.2 13 Feb 2023 -FIXES: +### Fixes - [3519](https://github.com/nginxinc/kubernetes-ingress/pull/3519) Add OnDelete to allowed strategy values - [3541](https://github.com/nginxinc/kubernetes-ingress/pull/3541) Ensure non-ready endpoints are not added to upstreams - [3527](https://github.com/nginxinc/kubernetes-ingress/pull/3527) Fix controller.topologySpreadConstraints schema, thanks to [Marco Londero](https://github.com/marcuz) -HELM CHART: - -- The version of the Helm chart is now 0.16.2. - -UPGRADE: +### Upgrade - For NGINX, use the 3.0.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.0.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.0.2 images from the F5 Container registry or build your own image using the 3.0.2 source code. - For Helm, use version 0.16.2 of the chart. -## NGINX Ingress Controller 3.0.1 +
+ +## 3.0.1 25 Jan 2023 -FIXES: +### Fixes - [3448](https://github.com/nginxinc/kubernetes-ingress/pull/3448) Fix Helm Chart Schema for priorityClassName - [3449](https://github.com/nginxinc/kubernetes-ingress/pull/3449) Correct nodeSelector in the Helm Chart Schema - [3463](https://github.com/nginxinc/kubernetes-ingress/pull/3463) Support non-VS created Challenge Ingress - [3481](https://github.com/nginxinc/kubernetes-ingress/pull/3481) Add missing OSS internal routes for NGINX Service Mesh -HELM CHART: - -- The version of the Helm chart is now 0.16.1. - -UPGRADE: +### Upgrade - For NGINX, use the 3.0.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.0.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 3.0.1 images from the F5 Container registry or build your own image using the 3.0.1 source code. - For Helm, use version 0.16.1 of the chart. -## NGINX Ingress Controller 3.0.0 +
+ +## 3.0.0 12 January 2023 -OVERVIEW: +### Overview - Added support for [Deep Service Insight](https://docs.nginx.com/nginx-ingress-controller/logging-and-monitoring/service-insight) for VirtualServer and TransportServer using the [-enable-service-insight](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-enable-service-insight) cli argument. - *The minimum supported version of Kubernetes is now 1.21*. NGINX Ingress Controller 3.0.0 removes support for `k8s.io/v1/Endpoints` API in favor of `discovery.k8s.io/v1/EndpointSlices`. For older Kubernetes versions, use the 2.4.x release of the Ingress Controller. @@ -359,18 +357,18 @@ OVERVIEW: - Beginning with NGINX Service Mesh release 1.7 it will include support for the free version of NGINX Ingress Controller as well as the paid version. - NGINX Ingress Controller + NGINX App Protect Denial of Service is now available through the AWS Marketplace. -FEATURES: +### Breaking Changes - [3260](https://github.com/nginxinc/kubernetes-ingress/pull/3260) Added support for EndpointSlices. + +### Features + - [3299](https://github.com/nginxinc/kubernetes-ingress/pull/3299) Support Dynamic namespaces using Labels. - [3261](https://github.com/nginxinc/kubernetes-ingress/pull/3261) Deep service insight endpoint for VirtualServer CR. - [3361](https://github.com/nginxinc/kubernetes-ingress/pull/3361) Added healthcheck for TransportServer CR. - [3347](https://github.com/nginxinc/kubernetes-ingress/pull/3347) Import JWKS from URL on JWT policy. - [3274](https://github.com/nginxinc/kubernetes-ingress/pull/3274) Allow configuration of map-hash-bucket-size and map-hash-max-size directives. - [3376](https://github.com/nginxinc/kubernetes-ingress/pull/3376) NGINX Service Mesh will support the free version of NGINX Ingress Controller when using NGINX open source. - -IMPROVEMENTS: - - [3170](https://github.com/nginxinc/kubernetes-ingress/pull/3170) Watch subset of namespaces for secrets. Thanks to [Hans Feldt](https://github.com/hafe). - [3341](https://github.com/nginxinc/kubernetes-ingress/pull/3341) Set value of `$remote_addr` to client IP when TLSPassthrough and Proxy Protocol are enabled. - [3131](https://github.com/nginxinc/kubernetes-ingress/pull/3131) NAP DoS images are now available in the AWS Marketplace. @@ -378,11 +376,11 @@ IMPROVEMENTS: - [2735](https://github.com/nginxinc/kubernetes-ingress/pull/2735) Support default client proxy headers to be overwritten in VirtualServer. Thanks to [Alex Wied](https://github.com/centromere). - [3133](https://github.com/nginxinc/kubernetes-ingress/pull/3133) Added caseSensitiveHttpHeaders to APPolicy CRD. Thanks to [Pavel Galitskiy](https://github.com/galitskiy). -FIXES: +### Fixes - [3139](https://github.com/nginxinc/kubernetes-ingress/pull/3139) Remove all IPV6 listeners in ingress resources with `-disable-ipv6` command line. -HELM CHART: +### Helm Chart - [3113](https://github.com/nginxinc/kubernetes-ingress/pull/3113) Added JSON Schema. - [3143](https://github.com/nginxinc/kubernetes-ingress/pull/3143) Added annotations for deployment and daemonset. @@ -390,87 +388,83 @@ HELM CHART: - [3065](https://github.com/nginxinc/kubernetes-ingress/pull/3065) Added annotations to the service account. Thanks to [0m1xa](https://github.com/0m1xa). - [3276](https://github.com/nginxinc/kubernetes-ingress/pull/3276) Added horizontalpodautoscaler. Thanks to [Bryan Hendryx](https://github.com/coolbry95). -UPGRADE: +### Upgrade - Make sure the Kubernetes version is in the supported platforms listed below. - For NGINX, use the 3.0.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=3.0.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 3.0.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 3.0.0 source code. - For Helm, use version 0.16.0 of the chart. Helm does not upgrade the CRDs. If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.21-1.26. -## NGINX Ingress Controller 2.4.2 +
+ +## 2.4.2 30 Nov 2022 -CHANGES: +### Dependencies - Update NGINX Plus version to R28. - Update NGINX App Protect WAF version to 4.0. - Update NGINX App Protect DoS version to 3.1. -HELM CHART: - -- The version of the Helm chart is now 0.15.2. - -UPGRADE: +### Upgrade - For NGINX, use the 2.4.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.4.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 2.4.2 images from the F5 Container registry or build your own image using the 2.4.2 source code. - For Helm, use version 0.15.2 of the chart. -## NGINX Ingress Controller 2.4.1 +
+ +## 2.4.1 19 October 2022 -CHANGES: +### Dependencies - [3183](https://github.com/nginxinc/kubernetes-ingress/pull/3183) Update NGINX version to 1.23.2. - [3175](https://github.com/nginxinc/kubernetes-ingress/pull/3175) Update Go dependencies. - Update NGINX Plus version to R27 P1. -FIXES: +### Fixes - [3139](https://github.com/nginxinc/kubernetes-ingress/pull/3139) Remove all IPV6 listeners in ingress resources with -disable-ipv6 command line. -HELM CHART: - -- The version of the Helm chart is now 0.15.1. - -UPGRADE: +### Upgrade - For NGINX, use the 2.4.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.4.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 2.4.1 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.4.1 source code. - For Helm, use version 0.15.1 of the chart. -## NGINX Ingress Controller 1.12.5 +
+ +## 1.12.5 19 October 2022 -CHANGES: +### Dependencies - Update NGINX version to 1.23.2. - Update NGINX Plus version to R27 P1. - Update Alpine to 3.16. - Update Go to 1.19 and Go dependencies. -HELM CHART: - -- The version of the Helm chart is now 0.10.5. - -UPGRADE: +### Upgrade - For NGINX, use the 1.12.5 image from our DockerHub: `nginx/nginx-ingress:1.12.5`, `nginx/nginx-ingress:1.12.5-alpine` or `nginx/nginx-ingress:1.12.5-ubi` - For NGINX Plus, please build your own image using the 1.12.5 source code. - For Helm, use version 0.10.5 of the chart. -## NGINX Ingress Controller 2.4.0 +
+ +## 2.4.0 04 October 2022 -OVERVIEW: +### Overview - Added support for enabling [proxy_protocol](https://github.com/nginxinc/kubernetes-ingress/tree/v2.4.0/examples/shared-examples/proxy-protocol) when port 443 is being used for both HTTPS traffic and [TLS Passthrough traffic](https://github.com/nginxinc/kubernetes-ingress/tree/v2.4.0/examples/custom-resources/tls-passthrough). - Updates to the TransportServer resource to support using [ExternalName services](https://kubernetes.io/docs/concepts/services-networking/service/#externalname). For examples, see [externalname-services](https://github.com/nginxinc/kubernetes-ingress/tree/v2.4.0/examples/custom-resources/externalname-services). @@ -480,7 +474,7 @@ OVERVIEW: - A new cli argument has been added: [-include-year](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-include-year). This appends the current year to the log output from the Ingress Controller. Example output: `I20220512 09:20:42.345457`. - Post-startup configuration reloads have been optimized to reduce traffic impacts. When many resources are modified at the same time, changes are combined to reduce the number of data plane reloads. -FEATURES: +### Features - [2986](https://github.com/nginxinc/kubernetes-ingress/pull/2986) Batch reloads at runtime. - [2914](https://github.com/nginxinc/kubernetes-ingress/pull/2914) Support watching multiple namespaces. @@ -488,18 +482,15 @@ FEATURES: - [2993](https://github.com/nginxinc/kubernetes-ingress/pull/2993) Accept proxy protocol when TLS passthrough enabled. - [3041](https://github.com/nginxinc/kubernetes-ingress/pull/3041) Support external name service for TransportServer. - [2939](https://github.com/nginxinc/kubernetes-ingress/pull/2939) Add support for wildcard hostname in VirtualServer. - -IMPROVEMENTS: - - [3040](https://github.com/nginxinc/kubernetes-ingress/pull/3040) Add command line argument to manually disable IPV6 listeners for unsupported clusters. - [3088](https://github.com/nginxinc/kubernetes-ingress/pull/3088) Filter secrets of type helm.sh/release.v1. -FIXES: +### Fixes - [2971](https://github.com/nginxinc/kubernetes-ingress/pull/2971) fix: Correct error message on missing path in path validation. Thanks to [Zachary Seguin](https://github.com/zachomedia). - [3095](https://github.com/nginxinc/kubernetes-ingress/pull/3095) do not create configmap if customConfigMap is used. Thanks to [Bryan Hendryx](https://github.com/coolbry95). -HELM CHART: +### Helm Chart - [3087](https://github.com/nginxinc/kubernetes-ingress/pull/3087) Allow omitting the default server secret from Helm installs. - [2831](https://github.com/nginxinc/kubernetes-ingress/pull/2831) Add ServiceMonitor to Helm Chart. Thanks to [araineUnity](https://github.com/araineUnity). @@ -507,48 +498,51 @@ HELM CHART: - [2979](https://github.com/nginxinc/kubernetes-ingress/pull/2979) Allow to specify image with digest in helm chart. Thanks to [Hans Feldt](https://github.com/hafe). - [3031](https://github.com/nginxinc/kubernetes-ingress/pull/3031) Adding automountServiceAccountToken to helm chart. -UPGRADE: +### Upgrade - For NGINX, use the 2.4.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.4.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.4.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.4.0 source code. - For Helm, use version 0.15.0 of the chart. If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs will not be upgraded. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and that passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.25. -## NGINX Ingress Controller 2.3.1 +
+ +## 2.3.1 16 September 2022 -CHANGES: +### Dependencies - [3048](https://github.com/nginxinc/kubernetes-ingress/pull/3048) Bump NGINX to 1.23.1 - [3049](https://github.com/nginxinc/kubernetes-ingress/pull/3049) Update Go dependencies. -- [3031](https://github.com/nginxinc/kubernetes-ingress/pull/3031) Add automountServiceAccountToken to helm chart -HELM CHART: +### Helm Chart -- The version of the Helm chart is now 0.14.1. +- [3031](https://github.com/nginxinc/kubernetes-ingress/pull/3031) Add automountServiceAccountToken to helm chart -UPGRADE: +### Upgrade - For NGINX, use the 2.3.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.3.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress), [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress) or [Quay.io](https://quay.io/repository/nginx/nginx-ingress). - For NGINX Plus, use the 2.3.1 images from the F5 Container registry or build your own image using the 2.3.1 source code. - For Helm, use version 0.14.1 of the chart. -## NGINX Ingress Controller 2.3.0 +
+ +## 2.3.0 12 July 2022 -OVERVIEW: +### Overview - Support making VirtualServer resources discoverable via public DNS servers using [external-dns](https://kubernetes-sigs.github.io/external-dns). Examples for configuring external-dns with NGINX Ingress Controller can be found [here](https://github.com/nginxinc/kubernetes-ingress/tree/v2.3.0/examples/custom-resources/external-dns). - Resolves [CVE-2022-30535](https://support.f5.com/csp/article/K52125139). This vulnerability impacted the visibility of secrets accessible by NGINX Ingress Controller. In some cases, secrets visible to NGINX Ingress Controller could be exposed to any authenticated user with permission to create and update Ingress objects. This vulnerability affected Ingress objects only - our Custom Resources (VirtualServer and TransportServer) were not affected. Customers unable to upgrade should migrate any Ingress resources to VirtualServer resources where possible, and use RBAC to restrict write access for users for Ingress objects. - Support using HTTP basic authentication with [VirtualServer](https://github.com/nginxinc/kubernetes-ingress/tree/v2.3.1/examples/custom-resources/basic-auth) and [Ingress](https://github.com/nginxinc/kubernetes-ingress/tree/v2.3.0/examples/basic-auth) resources. Special thanks to [Simon Wachter](https://github.com/svvac). - Support HTTP01 type ACME Issuers for use with VirtualServer resources with [cert-manager](https://cert-manager.io/docs/). -FEATURES: +### Features - [2581](https://github.com/nginxinc/kubernetes-ingress/pull/2581) Add OpenTracing to all Debian and Alpine based images. - [2328](https://github.com/nginxinc/kubernetes-ingress/pull/2328) Add handling of multiple log destinations. @@ -558,9 +552,6 @@ FEATURES: - [2801](https://github.com/nginxinc/kubernetes-ingress/pull/2801) Add SBOMs to release. - [2269](https://github.com/nginxinc/kubernetes-ingress/pull/2269) HTTP basic auth support. Thanks to [Simon Wachter](https://github.com/svvac). - [2800](https://github.com/nginxinc/kubernetes-ingress/pull/2800) Integrate external-dns with VirtualServer resources. - -IMPROVEMENTS: - - [2583](https://github.com/nginxinc/kubernetes-ingress/pull/2583) Add runAsNonRoot in deployments. - [2484](https://github.com/nginxinc/kubernetes-ingress/pull/2484) Add container resource requests. - [2627](https://github.com/nginxinc/kubernetes-ingress/pull/2627) Update InternalRoute server_name. @@ -575,7 +566,7 @@ IMPROVEMENTS: - [2783](https://github.com/nginxinc/kubernetes-ingress/pull/2783) Update validation regex for path spec. - [2781](https://github.com/nginxinc/kubernetes-ingress/pull/2781) Report Hostname in ExternalEndpoint for VS and VSR resources. -FIXES: +### Fixes - [2617](https://github.com/nginxinc/kubernetes-ingress/pull/2617) Fix Dockerfile for amd64 microarchitectures. - [2637](https://github.com/nginxinc/kubernetes-ingress/pull/2637) Add terminationGracePeriodSeconds to deployment. Thanks to [Maksym Iv](https://github.com/maksym-iv). @@ -583,68 +574,66 @@ FIXES: - [2673](https://github.com/nginxinc/kubernetes-ingress/pull/2673) Fix status.loadbalancer.hostname deletion on OOMKill. Thanks to [Heiko Voigt](https://github.com/hvoigt). - [2718](https://github.com/nginxinc/kubernetes-ingress/pull/2718) Fix cases where CM enabled but no TLS block specified in VS. -HELM CHART: +### Helm Chart - [2418](https://github.com/nginxinc/kubernetes-ingress/pull/2418) Add support for allocateLoadBalancerNodePorts, ipFamilyPolicy and ipFamilies. Thanks to [centromere](https://github.com/centromere). - [2672](https://github.com/nginxinc/kubernetes-ingress/pull/2672) Add minReadySeconds & strategy support. Thanks to [Ciaran](https://github.com/cmk-pcs). - [2625](https://github.com/nginxinc/kubernetes-ingress/pull/2625) allow configuring topologySpreadConstraints in Helm chart. Thanks to [Kamil Domański](https://github.com/kdomanski). -UPGRADE: +### Upgrade - For NGINX, use the 2.3.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.3.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.3.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.3.0 source code. - For Helm, use version 0.14.0 of the chart. If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs will not be upgraded. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. - When upgrading using the [manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/), make sure to update the [ClusterRole](https://github.com/nginxinc/kubernetes-ingress/blob/v2.3.1/deployments/rbac/rbac.yaml). This is required to enable the ExternalDNS for VirtualServer resources integration. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.24. -## NGINX Ingress Controller 2.2.2 +
+ +## 2.2.2 23 May 2022 -FIXES: +### Fixes - [2627](https://github.com/nginxinc/kubernetes-ingress/pull/2627) Update InternalRoute server_name. -HELM CHART: - -- The version of the Helm chart is now 0.13.2. - -UPGRADE: +### Upgrade - For NGINX, use the 2.2.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.2.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.2.2 images from the F5 Container registry or build your own image using the 2.2.2 source code. - For Helm, use version 0.13.2 of the chart. -## NGINX Ingress Controller 2.2.1 +
+ +## 2.2.1 17 May 2022 -CHANGES: +### Dependencies - Update Go dependencies. -FIXES: +### Fixes - [2654](https://github.com/nginxinc/kubernetes-ingress/pull/2654) Sync changes from [nginx-openid-connect](https://github.com/nginxinc/nginx-openid-connect) repo, add zoneSyncLeeway field in policy. For more information on the fixes, see [52](https://github.com/nginxinc/nginx-openid-connect/pull/52). -HELM CHART: - -- The version of the Helm chart is now 0.13.1. - -UPGRADE: +### Upgrade - For NGINX, use the 2.2.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.2.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.2.1 images from the F5 Container registry or build your own image using the 2.2.1 source code. - For Helm, use version 0.13.1 of the chart. -## NGINX Ingress Controller 2.2.0 +
+ +## 2.2.0 12 April 2022 -OVERVIEW: +### Overview - Support for automatic provisioning and management of Certificate resources for VirtualServer resources using [cert-manager](https://cert-manager.io/docs/). Examples for configuring cert-manager with NGINX Ingress Controller can be found [here](https://github.com/nginxinc/kubernetes-ingress/tree/v2.2.0/examples/custom-resources/certmanager). Please note that HTTP01 type ACME Issuers are not yet supported for use with VirtualServer resources. @@ -654,36 +643,30 @@ OVERVIEW: - A new [-enable-oidc](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/command-line-arguments/#-enable-oidc) cli argument has been added to enable OIDC policies. Previously, this behaviour was achieved through the usage of the `-enable-preview-policies` cli argument. -FEATURES: +### Features - [2576](https://github.com/nginxinc/kubernetes-ingress/pull/2576) Add support for IPv6. - [2572](https://github.com/nginxinc/kubernetes-ingress/pull/2572) Automate provisioning of Certificate resources for VirtualServer resources using cert-manager. - -IMPROVEMENTS: - - [2346](https://github.com/nginxinc/kubernetes-ingress/pull/2346) Use os.ReadDir for lightweight directory reading. Thanks to [Eng Zer Jun](https://github.com/Juneezee). - [2360](https://github.com/nginxinc/kubernetes-ingress/pull/2360) Add NGINX App Protect reconnect period directive. - [2479](https://github.com/nginxinc/kubernetes-ingress/pull/2479) Add cli argument to configure NGINX App Protect log level. - [2455](https://github.com/nginxinc/kubernetes-ingress/pull/2455) Increase memory available for NGINX App Protect xml parser. - -CHANGES: - - [2580](https://github.com/nginxinc/kubernetes-ingress/pull/2580) Create -enable-oidc command line argument for OIDC policy. - [2566](https://github.com/nginxinc/kubernetes-ingress/pull/2566) Unbind policy from preview policies. - [2582](https://github.com/nginxinc/kubernetes-ingress/pull/2582) Rename Make targets from `openshift` to `ubi`. -FIXES: +### Fixes - [2378](https://github.com/nginxinc/kubernetes-ingress/pull/2378) Fix healthcheck ports. - [2404](https://github.com/nginxinc/kubernetes-ingress/pull/2404) Start nginx with -e stderr parameter. - [2414](https://github.com/nginxinc/kubernetes-ingress/pull/2414) Fix in file nginx-plus.virtualserver.tmpl ApDosMonitor->ApDosMonitorURI. -HELM CHART: +### Helm Chart - [2525](https://github.com/nginxinc/kubernetes-ingress/pull/2525) Extend helm chart to include NGINX Service Mesh fields. - [2294](https://github.com/nginxinc/kubernetes-ingress/pull/2294) Add extra containers to helm chart. Thanks to [Márk Sági-Kazár](https://github.com/sagikazarmark). -UPGRADE: +### Upgrade - For NGINX, use the 2.2.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.2.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.2.0 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.2.0 source code. @@ -692,33 +675,33 @@ UPGRADE: - The -enable-preview-policies cli argument has been deprecated, and is no longer required for any Policy resources. - Enabling OIDC Policies now requires the use of -enable-oidc cli argument instead of the -enable-preview-policies cli argument. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.23. -## NGINX Ingress Controller 2.1.2 +
+ +## 2.1.2 29 March 2022 -CHANGES: +### Dependencies - Update UBI based images to 8. -HELM CHART: - -- The version of the Helm chart is now 0.12.2. - -UPGRADE: +### Upgrade - For NGINX, use the 2.1.2 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.1.2), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.1.2 images from the F5 Container registry or the [AWS Marketplace](https://aws.amazon.com/marketplace/search/?CREATOR=741df81b-dfdc-4d36-b8da-945ea66b522c&FULFILLMENT_OPTION_TYPE=CONTAINER&filters=CREATOR%2CFULFILLMENT_OPTION_TYPE) or build your own image using the 2.1.2 source code. - For Helm, use version 0.12.2 of the chart. -## NGINX Ingress Controller 1.12.4 +
+ +## 1.12.4 23 March 2022 -CHANGES: +### Dependencies - Update NGINX version to 1.21.6. - Update NGINX Plus version to R26. @@ -727,44 +710,40 @@ CHANGES: - Update UBI to 8. - Update Go to 1.17 and Go dependencies. -FIXES: +### Fixes - Fix OpenTracing not working with NGINX Plus. -HELM CHART: - -- The version of the Helm chart is now 0.10.4. - -UPGRADE: +### Upgrade - For NGINX, use the 1.12.4 image from our DockerHub: `nginx/nginx-ingress:1.12.4`, `nginx/nginx-ingress:1.12.4-alpine` or `nginx/nginx-ingress:1.12.4-ubi` - For NGINX Plus, please build your own image using the 1.12.4 source code. - For Helm, use version 0.10.4 of the chart. -## NGINX Ingress Controller 2.1.1 +
+ +## 2.1.1 17 February 2022 -CHANGES: +### Dependencies - Update NGINX version to 1.21.6. - Update NGINX Plus version to R26. -HELM CHART: - -- The version of the Helm chart is now 0.12.1. - -UPGRADE: +### Upgrade - For NGINX, use the 2.1.1 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.1.1), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.1.1 images from the F5 Container registry or build your own image using the 2.1.1 source code. - For Helm, use version 0.12.1 of the chart. -## NGINX Ingress Controller 2.1.0 +
+ +## 2.1.0 06 January 2022 -OVERVIEW: +### Overview - Support for NGINX App Protect Denial of Service protection with NGINX Ingress Controller. More information about [NGINX App Protect DoS](https://www.nginx.com/products/nginx-app-protect/denial-of-service/). Examples for configuring NGINX App Protect DoS with NGINX Ingress Controller can be found [here](https://github.com/nginxinc/kubernetes-ingress/tree/v2.1.1/examples/appprotect-dos). @@ -775,7 +754,7 @@ Mandatory health checks can be marked as persistent, so that the previous state See the settings [here](https://docs.nginx.com/nginx-ingress-controller/configuration/virtualserver-and-virtualserverroute-resources/#upstreamhealthcheck). More about the [NGINX Plus mandatory and persistent health check features](https://docs.nginx.com/nginx/admin-guide/load-balancer/http-health-check/#mandatory-health-checks) -FEATURES: +### Features - [2251](https://github.com/nginxinc/kubernetes-ingress/pull/2251) Enable setting mandatory and persistent in upstream healthchecks in VS and VSR. - [2241](https://github.com/nginxinc/kubernetes-ingress/pull/2241) Add support for NGINX App Protect DoS. @@ -786,16 +765,14 @@ FEATURES: - [2100](https://github.com/nginxinc/kubernetes-ingress/pull/2100) Add support for initContainers. Thanks to [Gunnar Scherf](https://github.com/g10f). - [1827](https://github.com/nginxinc/kubernetes-ingress/pull/1827) Add support for wildcard cert in VirtualServer resources. Thanks to [Simon Wachter](https://github.com/svvac). - [2107](https://github.com/nginxinc/kubernetes-ingress/pull/2107) Add option to download the NGINX Ingress Controller binary. Introduced a new `TARGET` `download` in the `Makefile` which can be used when building the NGINX Ingress Controller Docker image. With this option the Ingress Controller binary will be downloaded instead of built from source. - -IMPROVEMENTS: - - [2044](https://github.com/nginxinc/kubernetes-ingress/pull/2044) Upload NGINX Ingress Controller binaries to release. - [2094](https://github.com/nginxinc/kubernetes-ingress/pull/2094) AP: update appolicies crd. - [2216](https://github.com/nginxinc/kubernetes-ingress/pull/2216) Add grpc_status to the logs. - [2237](https://github.com/nginxinc/kubernetes-ingress/pull/2237) Unbind app-protect from -preview-policies. -- [2273](https://github.com/nginxinc/kubernetes-ingress/pull/2273) Make the resource comparison more informative in case of an error. Thanks to [Andrey Karpov](https://github.com/ndk) +- [2273](https://github.com/nginxinc/kubernetes-ingress/pull/2273) Make the resource comparison more informative in case of an error. Thanks to [Andrey Karpov](https://github.com/ndk). +- [2124](https://github.com/nginxinc/kubernetes-ingress/pull/2124) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. -FIXES: +### Fixes - [2267](https://github.com/nginxinc/kubernetes-ingress/pull/2267) Fix URI rewrite in VirtualServers and VirtualServerRoutes. - [2260](https://github.com/nginxinc/kubernetes-ingress/pull/2260) Check if refresh token is `undefined` and do not store it in this case. Thanks to [tippexs](https://github.com/tippexs) for the fix. @@ -804,134 +781,122 @@ FIXES: - [2125](https://github.com/nginxinc/kubernetes-ingress/pull/2125) Allow empty string in server-tokens annotation for NGINX Plus. - [2042](https://github.com/nginxinc/kubernetes-ingress/pull/2042) Use release specific repo for NGINX Plus on Debian. -CHANGES: +### Dependencies -- [2124](https://github.com/nginxinc/kubernetes-ingress/pull/2124) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. - [2173](https://github.com/nginxinc/kubernetes-ingress/pull/2173) Update Debian to Bullseye. - Update NGINX Plus version to R25. - Update NGINX version to 1.21.5. -HELM CHART: - -- The version of the Helm chart is now 0.12.0. - -UPGRADE: +### Upgrade - For NGINX, use the 2.1.0 images from our [DockerHub](https://hub.docker.com/r/nginx/nginx-ingress/tags?page=1&ordering=last_updated&name=2.1.0), [GitHub Container](https://github.com/nginxinc/kubernetes-ingress/pkgs/container/kubernetes-ingress) or [Amazon ECR Public Gallery](https://gallery.ecr.aws/nginx/nginx-ingress). - For NGINX Plus, use the 2.1.0 images from the F5 Container registry or build your own image using the 2.1.0 source code. - For Helm, use version 0.12.0 of the chart. - We changed the behaviour of snippets in Ingress resources by extending the existing -enable-snippets cli argument to apply to Ingress resources as well as VirtualServer, VirtualServerRoute and TransportServer resources. Because the default value of -enable-snippets is false, if you are using snippets in Ingress resources, you must explicitly set the -enable-snippets to true before upgrading the Ingress Controller, so that the new version of the Ingress Controller doesn't reject Ingresses with snippets annotations. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.23. -## NGINX Ingress Controller 2.0.3 +
+ +## 2.0.3 28 October 2021 -CHANGES: +### Breaking Changes - [2124](https://github.com/nginxinc/kubernetes-ingress/pull/2124) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. -- [2132](https://github.com/nginxinc/kubernetes-ingress/pull/2132) Install libcurl on OpenTracing for NGINX Plus. -HELM CHART: +### Fixes -- The version of the Helm chart is now 0.11.3. +- [2132](https://github.com/nginxinc/kubernetes-ingress/pull/2132) Install libcurl on OpenTracing for NGINX Plus. -UPGRADE: +### Upgrade - For NGINX, use the 2.0.3 image from our DockerHub: `nginx/nginx-ingress:2.0.3`, `nginx/nginx-ingress:2.0.3-alpine` or `nginx/nginx-ingress:2.0.3-ubi` - For NGINX Plus, please build your own image using the 2.0.3 source code. - For Helm, use version 0.11.3 of the chart. - We changed the behaviour of snippets in Ingress resources by extending the existing -enable-snippets cli argument to apply to Ingress resources as well as VirtualServer, VirtualServerRoute and TransportServer resources. Because the default value of -enable-snippets is false, if you are using snippets in Ingress resources, you must explicitly set the -enable-snippets to true before upgrading the Ingress Controller, so that the new version of the Ingress Controller doesn't reject Ingresses with snippets annotations. -## NGINX Ingress Controller 1.12.3 +
+ +## 1.12.3 28 October 2021 -FIXES: +### Fixes - [2133](https://github.com/nginxinc/kubernetes-ingress/pull/2133) Use release specific repo for the App Protect packages on Debian. This fixes an error when building Debian-based images with NGINX Plus with App Protect: previously, building an image would fail with the error `nginx-plus-module-appprotect : Depends: app-protect-plugin (= 3.639.0-1~buster) but 3.671.0-1~buster is to be installed`. The bug first appeared when NGINX App Protect version 3.6 was released on 13 October 2021. - [2134](https://github.com/nginxinc/kubernetes-ingress/pull/2134) Apply -enable-snippets cli arg to Ingresses. This PR extends the existing -enable-snippets cli argument to apply to Ingress resources. If snippets are not enabled, the Ingress Controller will reject any Ingress resources with snippets annotations. Previously, the argument only applied to VirtualServer, VirtualServerRoute and TransportServer resources. Please Note: this is a breaking change. See the `UPGRADE` instructions below. -HELM CHART: - -- The version of the Helm chart is now 0.10.3. - -UPGRADE: +### Upgrade - For NGINX, use the 1.12.3 image from our DockerHub: `nginx/nginx-ingress:1.12.3`, `nginx/nginx-ingress:1.12.3-alpine` or `nginx/nginx-ingress:1.12.3-ubi` - For NGINX Plus, please build your own image using the 1.12.3 source code. - For Helm, use version 0.10.3 of the chart. - We changed the behaviour of snippets in Ingress resources by extending the existing -enable-snippets cli argument to apply to Ingress resources as well as VirtualServer, VirtualServerRoute and TransportServer resources. Because the default value of -enable-snippets is false, if you are using snippets in Ingress resources, you must explicitly set the -enable-snippets to true before upgrading the Ingress Controller, so that the new version of the Ingress Controller doesn't reject Ingresses with snippets annotations. -## NGINX Ingress Controller 2.0.2 +
+ +## 2.0.2 13 October 2021 -CHANGES: +### Dependencies - Update NGINX App Protect version to 3.6. - Update NGINX Plus version to R25 in NGINX App Protect enabled images. -- [2074](https://github.com/nginxinc/kubernetes-ingress/pull/2074) Update JWT library to `golang-jwt/jwt`. Previously, the Ingress Controller used `dgrijalva/jwt-go`, which has a vulnerability [CVE-2020-26160](https://nvd.nist.gov/vuln/detail/CVE-2020-26160). Note that the Ingress Controller wasn’t affected by this vulnerability, and the jwt library was used only in the NGINX Plus images from AWS Marketplace for Containers. -HELM CHART: +### Features -- The version of the Helm chart is now 0.11.2. +- [2074](https://github.com/nginxinc/kubernetes-ingress/pull/2074) Update JWT library to `golang-jwt/jwt`. Previously, the Ingress Controller used `dgrijalva/jwt-go`, which has a vulnerability [CVE-2020-26160](https://nvd.nist.gov/vuln/detail/CVE-2020-26160). Note that the Ingress Controller wasn’t affected by this vulnerability, and the jwt library was used only in the NGINX Plus images from AWS Marketplace for Containers. -UPGRADE: +### Upgrade - For NGINX, use the 2.0.2 image from our DockerHub. - For NGINX Plus, use the 2.0.2 from the F5 Container registry or build your own image using the 2.0.2 source code. - For Helm, use version 0.11.2 of the chart. -## NGINX Ingress Controller 2.0.1 +
+ +## 2.0.1 07 October 2021 -FIXES: +### Fixes - [2051](https://github.com/nginxinc/kubernetes-ingress/pull/2051) Use release specific repo for NGINX Plus on Debian. This fixes an error when building the Debian-based image with NGINX Plus and App Protect: previously, building the image would fail with the error `Package 'nginx-plus-r24' has no installation candidate`. -DOCUMENTATION IMPROVEMENTS: - -- [2059](https://github.com/nginxinc/kubernetes-ingress/pull/2059) fixes issues with the 404 and robots.txts redirects. -- [2049](https://github.com/nginxinc/kubernetes-ingress/pull/2049) Remove note from operator installation. A version of the Operator compatible with 2.0.x is now available. - -HELM CHART: - -- The version of the Helm chart is now 0.11.1. - -UPGRADE: +### Upgrade - For NGINX, use the 2.0.1 image from our DockerHub. - For NGINX Plus, use the 2.0.1 from the F5 Container registry or build your own image using the 2.0.1 source code. - For Helm, use version 0.11.1 of the chart. -## NGINX Ingress Controller 1.12.2 +
+ +## 1.12.2 7 October 2021 -FIXES: +### Fixes - [2048](https://github.com/nginxinc/kubernetes-ingress/pull/2048) Use release specific repo for NGINX Plus on Debian. This fixes an error when building Debian-based images with NGINX Plus: previously, building an image would fail with the error `Package 'nginx-plus-r24' has no installation candidate`. The bug first appeared when NGINX Plus R25 was released on 28 September 2021. -HELM CHART: - -- The version of the Helm chart is now 0.10.2. - -UPGRADE: +### Upgrade - For NGINX, use the 1.12.2 image from our DockerHub: `nginx/nginx-ingress:1.12.2`, `nginx/nginx-ingress:1.12.2-alpine` or `nginx/nginx-ingress:1.12.2-ubi` - For NGINX Plus, please build your own image using the 1.12.2 source code. - For Helm, use version 0.10.2 of the chart. -## NGINX Ingress Controller 2.0.0 +
+ +## 2.0.0 28 September 2021 -OVERVIEW: +### Overview Release 2.0.0 includes: @@ -948,24 +913,26 @@ Release 2.0.0 includes: You will find the complete changelog for release 2.0.0, including bug fixes, improvements, and changes below. -FEATURES: +### Breaking Changes -- [1908](https://github.com/nginxinc/kubernetes-ingress/pull/1908) Add NTLM support to VirtualServer and VirtualServerRoute upstreams. - [1850](https://github.com/nginxinc/kubernetes-ingress/pull/1850) Support Ingress and IngressClass v1. -- [1746](https://github.com/nginxinc/kubernetes-ingress/pull/1746) Add ingressClassName field to Policy. -IMPROVEMENTS: +### Features +- [1908](https://github.com/nginxinc/kubernetes-ingress/pull/1908) Add NTLM support to VirtualServer and VirtualServerRoute upstreams. +- [1746](https://github.com/nginxinc/kubernetes-ingress/pull/1746) Add ingressClassName field to Policy. - [1956](https://github.com/nginxinc/kubernetes-ingress/pull/1956) Add v1alpha1 version back to policy CRD. - [1907](https://github.com/nginxinc/kubernetes-ingress/pull/1907) Remove libs compilation for OpenTracing in Dockerfile; add Zipkin and Datadog in addition to the already supported Jaeger tracer; additionally, for NGINX we now publish a Docker image with the tracers and the OpenTracing module on DockerHub: `nginx-ic/nginx-plus-ingress:1.12.0-ot`. Also thanks to [MatyRi](https://github.com/MatyRi) for upgrading OpenTracing in [1883](https://github.com/nginxinc/kubernetes-ingress/pull/1883). - [1788](https://github.com/nginxinc/kubernetes-ingress/pull/1788) Reload only once during the start. This significantly reduces the time it takes for an Ingress Controller pod to become ready when hundreds of Ingress or other supported resources are created in the cluster. +- [1721](https://github.com/nginxinc/kubernetes-ingress/pull/1721) Increase default reload timeout to 60s: the Ingress Controller will wait for 60s for NGINX to start or reload. Previously, the default was 4 seconds. +- [2009](https://github.com/nginxinc/kubernetes-ingress/pull/2009) Increase default upstream zone size for NGINX Plus. See the INCREASED UPSTREAM ZONES section below. -FIXES: +### Fixes - [1926](https://github.com/nginxinc/kubernetes-ingress/pull/1926) Fix increased IC pod startup time when hundreds of VirtualServerRoutes are used - [1712](https://github.com/nginxinc/kubernetes-ingress/pull/1712) Allow `make` to build image when .git directory is missing. -DOCUMENTATION IMPROVEMENTS: +### Documentation - [1932](https://github.com/nginxinc/kubernetes-ingress/pull/1932) Add IAM instructions for NGINX Plus AWS Marketplace images. - [1927](https://github.com/nginxinc/kubernetes-ingress/pull/1927) Fix function name comments typo. Thanks to [Sven Nebel](https://github.com/snebel29). @@ -975,20 +942,17 @@ DOCUMENTATION IMPROVEMENTS: - [1740](https://github.com/nginxinc/kubernetes-ingress/pull/1740) Publish docs using Hugo and Netlify. - [1702](https://github.com/nginxinc/kubernetes-ingress/pull/1702) Add security recommendations documentation. -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.11.0. - Add new parameters to the Chart: `controller.pod.extraLabels`. Added in [1884](https://github.com/nginxinc/kubernetes-ingress/pull/1884). -CHANGES: +### Dependencies - [1855](https://github.com/nginxinc/kubernetes-ingress/pull/1855) Update minimum Kubernetes version to 1.19; remove the `-use-ingress-class-only` command-line argument, which doesn't work with Kubernetes >= 1.19. -- [1721](https://github.com/nginxinc/kubernetes-ingress/pull/1721) Increase default reload timeout to 60s: the Ingress Controller will wait for 60s for NGINX to start or reload. Previously, the default was 4 seconds. -- [2009](https://github.com/nginxinc/kubernetes-ingress/pull/2009) Increase default upstream zone size for NGINX Plus. See the INCREASED UPSTREAM ZONES section below. - Update NGINX Plus version to R25. **Note**: images with NGINX App Protect will continue to use R24 until App Protect 3.6 is released. - Update NGINX version to 1.21.3. -UPGRADE: +### Upgrade - For NGINX, use the 2.0.0 image from our DockerHub. - For NGINX Plus, use the 2.0.0 from the F5 Container registry or build your own image using the 2.0.0 source code. @@ -1020,34 +984,34 @@ Please also read the [release 1.10 changelog](https://docs.nginx.com/nginx-ingre Note that 2.1.0 will remove support for the v1alpha1 version of the Policy. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.19-1.22. -## NGINX Ingress Controller 1.12.1 +
+ +## 1.12.1 8 September 2021 -CHANGES: +### Dependencies - Update NGINX App Protect version to 3.5. -HELM CHART: - -- The version of the Helm chart is now 0.10.1. - -UPGRADE: +### Upgrade - For NGINX, use the 1.12.1 image from our DockerHub: `nginx/nginx-ingress:1.12.1`, `nginx/nginx-ingress:1.12.1-alpine` or `nginx/nginx-ingress:1.12.1-ubi` -- For NGINX Plus, use the 1.12.1 image from the F5 Container Registry - see [the documentation here]({{< relref "installation/nic-images/pulling-ingress-controller-image" >}}) +- For NGINX Plus, use the 1.12.1 image from the F5 Container Registry - see [the documentation here](https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/) - Alternatively, you can also build your own image using the 1.12.1 source code. - For Helm, use version 0.10.1 of the chart. -## NGINX Ingress Controller 1.12.0 +
+ +## 1.12.0 30 June 2021 -OVERVIEW: +### Overview Release 1.12.0 includes: @@ -1058,23 +1022,17 @@ Release 1.12.0 includes: You will find the complete changelog for release 1.12.0, including bug fixes, improvements, and changes below. -FEATURES: +### Features - [1633](https://github.com/nginxinc/kubernetes-ingress/pull/1633) Support match in TransportServer health checks. - [1619](https://github.com/nginxinc/kubernetes-ingress/pull/1619) Add AWS Marketplace Entitlement verification. - [1480](https://github.com/nginxinc/kubernetes-ingress/pull/1480) Add max connections to TransportServer. - [1479](https://github.com/nginxinc/kubernetes-ingress/pull/1479) Add load balancing method to TransportServer. - [1466](https://github.com/nginxinc/kubernetes-ingress/pull/1466) Support snippets in TransportServer. - -FEATURES FOR NGINX APP PROTECT: - - [1578](https://github.com/nginxinc/kubernetes-ingress/pull/1578) Add support for CSRF protection in APPolicy. - [1513](https://github.com/nginxinc/kubernetes-ingress/pull/1513) Support multiple log security configs in Ingresses. - [1481](https://github.com/nginxinc/kubernetes-ingress/pull/1481) Add support for user defined browsers in APPolicy. - [1411](https://github.com/nginxinc/kubernetes-ingress/pull/1411) Add unary gRPC support in APPolicy. - -IMPROVEMENTS: - - [1671](https://github.com/nginxinc/kubernetes-ingress/pull/1671) Simplify Dockerfile stages for Debian. - [1652](https://github.com/nginxinc/kubernetes-ingress/pull/1652) Add HTTPS option to Prometheus endpoint. - [1646](https://github.com/nginxinc/kubernetes-ingress/pull/1646) Improve Dockerfile. @@ -1083,98 +1041,91 @@ IMPROVEMENTS: - [1500](https://github.com/nginxinc/kubernetes-ingress/pull/1500) Support ssl_reject_handshake in Ingress and VS. - [1494](https://github.com/nginxinc/kubernetes-ingress/pull/1494) Add logs around NGINX Plus binary/flag mismatch. - [1492](https://github.com/nginxinc/kubernetes-ingress/pull/1492) Update the IC so that GlobalConfiguration is not mandatory when configured. +- [1500](https://github.com/nginxinc/kubernetes-ingress/pull/1500) Support ssl_reject_handshake in Ingress and VS. Previously, to handle missing or invalid TLS Secrets in Ingress and VirtualServer resources, the Ingress Controller would configure NGINX to break any attempts for clients to establish TLS connections to the affected hosts using `ssl_ciphers NULL;` in the NGINX configuration. The method didn't work for TLS v1.3. Now the Ingress Controller uses `ssl_reject_handshake on;`, which works for TLS v1.3. - Documentation improvements: [1649](https://github.com/nginxinc/kubernetes-ingress/pull/1649). -FIXES: +### Fixes - [1658](https://github.com/nginxinc/kubernetes-ingress/pull/1658) Add missing njs module to the openshift-image-nap-plus image. - [1654](https://github.com/nginxinc/kubernetes-ingress/pull/1654) Fix incorrect configuration and unexpected warnings about Secrets at the IC start. - [1501](https://github.com/nginxinc/kubernetes-ingress/pull/1501) Fix ungraceful shutdown of NGINX. - Documentation fixes: [1668](https://github.com/nginxinc/kubernetes-ingress/pull/1668), [1594](https://github.com/nginxinc/kubernetes-ingress/pull/1594) thanks to [shaggy245](https://github.com/shaggy245), [1563](https://github.com/nginxinc/kubernetes-ingress/pull/1563), [1551](https://github.com/nginxinc/kubernetes-ingress/pull/1551). -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.10.0. - Add new parameters to the Chart: `prometheus.scheme`, `prometheus.secret`. Added in [1652](https://github.com/nginxinc/kubernetes-ingress/pull/1652). -CHANGES: +### Dependencies - [1604](https://github.com/nginxinc/kubernetes-ingress/pull/1604) Update NGINX Plus to R24. Previously, the Dockerfile had a fixed NGINX Plus version. Now the Dockerfile has a floating version that corresponds to the latest major NGINX Plus version. In the event of a patch version of NGINX Plus being released, make sure to rebuild your image to get the latest version (previously, we released a new Ingress Controller release in that case). Additionally, the AppProtect related packages are no longer fixed -- the Dockerfile will always install the latest version of the packages that work with the latest NGINX Plus version. -- [1500](https://github.com/nginxinc/kubernetes-ingress/pull/1500) Support ssl_reject_handshake in Ingress and VS. Previously, to handle missing or invalid TLS Secrets in Ingress and VirtualServer resources, the Ingress Controller would configure NGINX to break any attempts for clients to establish TLS connections to the affected hosts using `ssl_ciphers NULL;` in the NGINX configuration. The method didn't work for TLS v1.3. Now the Ingress Controller uses `ssl_reject_handshake on;`, which works for TLS v1.3. -- Update NGINX Plus version to R24. - Update NGINX version to 1.21.0. -UPGRADE: +### Upgrade - For NGINX, use the 1.12.0 image from our DockerHub: `nginx/nginx-ingress:1.12.0`, `nginx/nginx-ingress:1.12.0-alpine` or `nginx-ingress:1.12.0-ubi` - For NGINX Plus, please build your own image using the 1.12.0 source code. - For Helm, use version 0.10.0 of the chart. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.16-1.21. -## NGINX Ingress Controller 1.11.3 +
+ +## 1.11.3 25 May 2021 -CHANGES: +### Dependencies - Update NGINX version to 1.21.0. -HELM CHART: - -- The version of the Helm chart is now 0.9.3. - -UPGRADE: +### Upgrade - For NGINX, use the 1.11.3 image from our DockerHub: `nginx/nginx-ingress:1.11.3`, `nginx/nginx-ingress:1.11.3-alpine` or `nginx/nginx-ingress:1.11.3-ubi` - For NGINX Plus, please build your own image using the 1.11.3 source code. - For Helm, use version 0.9.3 of the chart. -## NGINX Ingress Controller 1.11.2 +
+ +## 1.11.2 19 May 2021 -CHANGES: +### Dependencies - Update NGINX Plus version to R23 P1. -HELM CHART: - -- The version of the Helm chart is now 0.9.2. - -UPGRADE: +### Upgrade - For NGINX, use the 1.11.2 image from our DockerHub: `nginx/nginx-ingress:1.11.2`, `nginx/nginx-ingress:1.11.2-alpine` or `nginx/nginx-ingress:1.11.2-ubi` - For NGINX Plus, please build your own image using the 1.11.2 source code. - For Helm, use version 0.9.2 of the chart. -## NGINX Ingress Controller 1.11.1 +
+ +## 1.11.1 7 April 2021 -CHANGES: +### Dependencies - Update NGINX version to 1.19.9. - Update the OpenSSL libraries used in the UBI images -- Fixed documentation in the Helm README - -HELM CHART: -- The version of the Helm chart is now 0.9.1. - -UPGRADE: +### Upgrade - For NGINX, use the 1.11.1 image from our DockerHub: `nginx/nginx-ingress:1.11.1`, `nginx/nginx-ingress:1.11.1-alpine` or `nginx/nginx-ingress:1.11.1-ubi` - For NGINX Plus, please build your own image using the 1.11.1 source code. - For Helm, use version 0.9.1 of the chart. -## NGINX Ingress Controller 1.11.0 +
+ +## 1.11.0 31 March 2021 -OVERVIEW: +### Overview Release 1.11.0 includes: @@ -1184,7 +1135,7 @@ Release 1.11.0 includes: You will find the complete changelog for release 1.11.0, including bug fixes, improvements, and changes below. -FEATURES: +### Features - [1317](https://github.com/nginxinc/kubernetes-ingress/pull/1317) Add status field to Policy resource. - [1449](https://github.com/nginxinc/kubernetes-ingress/pull/1449) Add support for ClusterIP in upstreams in VirtualServers/VirtualServerRoutes. @@ -1194,13 +1145,7 @@ FEATURES: - [1382](https://github.com/nginxinc/kubernetes-ingress/pull/1382) Add passive health checks to TransportServer. - [1346](https://github.com/nginxinc/kubernetes-ingress/pull/1346) Add configurable timeouts to TransportServer. - [1297](https://github.com/nginxinc/kubernetes-ingress/pull/1297) Support custom return in the default server. Thanks to [030](https://github.com/030). - -FEATURES FOR NGINX APP PROTECT: - - [1378](https://github.com/nginxinc/kubernetes-ingress/pull/1378) Add WAF Policy. - -IMPROVEMENTS: - - [1420](https://github.com/nginxinc/kubernetes-ingress/pull/1420) Support IngressClassName in TransportServer. - [1415](https://github.com/nginxinc/kubernetes-ingress/pull/1415) Handle host and listener collisions for TransportServer resource. - [1322](https://github.com/nginxinc/kubernetes-ingress/pull/1322) Improve VirtualServer/VirtualServerRoute warnings for Policies. @@ -1208,64 +1153,63 @@ IMPROVEMENTS: - [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) Refactor Dockerfile and Makefile. - Documentation improvements: [1320](https://github.com/nginxinc/kubernetes-ingress/pull/1320), [1326](https://github.com/nginxinc/kubernetes-ingress/pull/1326), and [1377](https://github.com/nginxinc/kubernetes-ingress/pull/1377). -FIXES: +### Fixes - [1457](https://github.com/nginxinc/kubernetes-ingress/pull/1457) Wait for caches to sync when the Ingress Controller starts. - [1444](https://github.com/nginxinc/kubernetes-ingress/pull/1444) Fix setting host header in action proxy in VirtualServer/VirtualServerRoute. - [1396](https://github.com/nginxinc/kubernetes-ingress/pull/1396) Fix reload timeout calculation for verifying NGINX reloads. -HELM CHART: - -- The version of the helm chart is now 0.9.0. - -CHANGES: +### Dependencies - [1455](https://github.com/nginxinc/kubernetes-ingress/pull/1455) Update NGINX version to 1.19.8. - [1428](https://github.com/nginxinc/kubernetes-ingress/pull/1428) Update Nginx App Protect version to 3.0. **Note**: [The Advanced gRPC Protection for Unary Traffic](/nginx-app-protect/configuration/#advanced-grpc-protection-for-unary-traffic) is not currently supported. -KNOWN ISSUES: +### Known Issues - [1448](https://github.com/nginxinc/kubernetes-ingress/issues/1448) When an Ingress Controller pod starts, it can report warnings about missing secrets for Ingress and other resources that reference secrets. Those warnings are intermittent - once the Ingress Controller fully processes the resources of the cluster, it will clear the warnings. Only after that, the Ingress Controller will become ready to accept client traffic - its readiness probe will succeed. -UPGRADE: +### Upgrade - For NGINX, use the 1.11.0 image from our DockerHub: `nginx/nginx-ingress:1.11.0`, `nginx/nginx-ingress:1.11.0-alpine` or `nginx-ingress:1.11.0-ubi` - For NGINX Plus, please build your own image using the 1.11.0 source code. - For Helm, use version 0.9.0 of the chart. -- [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) improved the Makefile. As a result, the commands for building the Ingress Controller image were changed. See the updated commands [here]({{< relref "installation/building-nginx-ingress-controller#build-image-push-to-private-repo" >}}). +- [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) improved the Makefile. As a result, the commands for building the Ingress Controller image were changed. See the updated commands [here](https://docs.nginx.com/nginx-ingress-controller/installation/building-ingress-controller-image/#building-the-image-and-pushing-it-to-the-private-registry). - [1241](https://github.com/nginxinc/kubernetes-ingress/pull/1241) also consolidated all Dockerfiles into a single Dockerfile. If you customized any of the Dockerfiles, make sure to port the changes to the new Dockerfile. - [1288](https://github.com/nginxinc/kubernetes-ingress/pull/1288) further improved validation of Ingress annotations. See this [document](https://docs.nginx.com/nginx-ingress-controller/configuration/ingress-resources/advanced-configuration-with-annotations/#validation) to learn more about which annotations are validated. Note that the Ingress Controller will reject resources with invalid annotations, which means clients will see `404` responses from NGINX. Before upgrading, ensure the Ingress resources don't have annotations with invalid values. Otherwise, after the upgrade, the Ingress Controller will reject such resources. - [1457](https://github.com/nginxinc/kubernetes-ingress/pull/1457) fixed the bug when an Ingress Controller pod could become ready before it generated the configuration for all relevant resources in the cluster. The fix also requires that the Ingress Controller can successfully list the relevant resources from the Kubernetes API. For example, if the `-enable-custom-resources` cli argument is `true` (which is the default), the VirtualServer, VirtualServerRoute, TransportServer, and Policy CRDs must be created in the cluster, so that the Ingress Controller can list them. This is similar to other custom resources -- see the list [here](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/#create-custom-resources). Thus, before upgrading, make sure that the CRDs are created in the cluster. Otherwise, the Ingress Controller pods will not become ready. -SUPPORTED PLATFORMS: +### Supported Platforms We will provide technical support for the NGINX Ingress Controller on any Kubernetes platform that is currently supported by its provider and which passes the Kubernetes conformance tests. This release was fully tested on the following Kubernetes versions: 1.16-1.20. -## NGINX Ingress Controller 1.10.1 +
+ +## 1.10.1 16 March 2021 -CHANGES: +### Dependencies - Update NGINX version to 1.19.8. - Add Kubernetes 1.20 support. -- [1373](https://github.com/nginxinc/kubernetes-ingress/pull/1373), [1439](https://github.com/nginxinc/kubernetes-ingress/pull/1439), [1440](https://github.com/nginxinc/kubernetes-ingress/pull/1440): Fix various issues in the Makefile. In 1.10.0, a bug was introduced that prevented building Ingress Controller images on versions of make < 4.1. -HELM CHART: +### Fixes -- The version of the Helm chart is now 0.8.1. +- [1373](https://github.com/nginxinc/kubernetes-ingress/pull/1373), [1439](https://github.com/nginxinc/kubernetes-ingress/pull/1439), [1440](https://github.com/nginxinc/kubernetes-ingress/pull/1440): Fix various issues in the Makefile. In 1.10.0, a bug was introduced that prevented building Ingress Controller images on versions of make < 4.1. -UPGRADE: +### Upgrade - For NGINX, use the 1.10.1 image from our DockerHub: `nginx/nginx-ingress:1.10.1`, `nginx/nginx-ingress:1.10.1-alpine` or `nginx/nginx-ingress:1.10.1-ubi` - For NGINX Plus, please build your own image using the 1.10.1 source code. - For Helm, use version 0.8.1 of the chart. -## NGINX Ingress Controller 1.10.0 +
+ +## 1.10.0 26 January 2021 -OVERVIEW: +### Overview Release 1.10.0 includes: @@ -1277,46 +1221,36 @@ Release 1.10.0 includes: You will find the complete changelog for release 1.10.0, including bug fixes, improvements, and changes below. -FEATURES FOR POLICY RESOURCE: +### Features - [1304](https://github.com/nginxinc/kubernetes-ingress/pull/1304) Add Open ID Connect policy. - -FEATURES FOR NGINX APP PROTECT: - - [1281](https://github.com/nginxinc/kubernetes-ingress/pull/1281) Add support for App Protect User Defined Signatures. - -FEATURES: - - [1266](https://github.com/nginxinc/kubernetes-ingress/pull/1266) Add workqueue metrics to Prometheus metrics. - [1233](https://github.com/nginxinc/kubernetes-ingress/pull/1233) Annotate tcp metrics with k8s object labels. - [1231](https://github.com/nginxinc/kubernetes-ingress/pull/1231) Support k8s objects variables in log format. - -IMPROVEMENTS: - - [1270](https://github.com/nginxinc/kubernetes-ingress/pull/1270) and [1277](https://github.com/nginxinc/kubernetes-ingress/pull/1277) Improve validation of Ingress annotations. - [1265](https://github.com/nginxinc/kubernetes-ingress/pull/1265) Report warnings for misconfigured TLS and JWK secrets. - [1262](https://github.com/nginxinc/kubernetes-ingress/pull/1262) Use setcap(8) only once. [1263](https://github.com/nginxinc/kubernetes-ingress/pull/1263) Use chown(8) only once. [1264](https://github.com/nginxinc/kubernetes-ingress/pull/1264) Use mkdir(1) only once. Thanks to [Sergey A. Osokin](https://github.com/osokin). - [1256](https://github.com/nginxinc/kubernetes-ingress/pull/1256) and [1260](https://github.com/nginxinc/kubernetes-ingress/pull/1260) Improve handling of secret resources. - [1240](https://github.com/nginxinc/kubernetes-ingress/pull/1240) Validate TLS and CA secrets. - [1235](https://github.com/nginxinc/kubernetes-ingress/pull/1235) Use buildkit secret flag for NGINX plus images. +- [1290](https://github.com/nginxinc/kubernetes-ingress/pull/1290) Graduate policy resource and accessControl policy to generally available. +- [1225](https://github.com/nginxinc/kubernetes-ingress/pull/1225) Require secrets to have types. +- [1237](https://github.com/nginxinc/kubernetes-ingress/pull/1237) Deprecate support for helm2 clients. - Documentation improvements: [1282](https://github.com/nginxinc/kubernetes-ingress/pull/1282), [1293](https://github.com/nginxinc/kubernetes-ingress/pull/1293), [1303](https://github.com/nginxinc/kubernetes-ingress/pull/1303), [1315](https://github.com/nginxinc/kubernetes-ingress/pull/1315). -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.8.0. - [1290](https://github.com/nginxinc/kubernetes-ingress/pull/1290) Add new preview policies parameter to chart. `controller.enablePreviewPolicies` was added. - [1232](https://github.com/nginxinc/kubernetes-ingress/pull/1232) Replace deprecated imagePullSecrets helm setting. `controller.serviceAccount.imagePullSecrets` was removed. `controller.serviceAccount.imagePullSecretName` was added. - [1228](https://github.com/nginxinc/kubernetes-ingress/pull/1228) Fix installation of ingressclass on Kubernetes versions `v1.18.x-*` -CHANGES: +### Dependencies - [1299](https://github.com/nginxinc/kubernetes-ingress/pull/1299) Update NGINX App Protect version to 2.3 and debian distribution to `debian:buster-slim`. - [1291](https://github.com/nginxinc/kubernetes-ingress/pull/1291) Update NGINX OSS to `1.19.6`. Update NGINX Plus to `R23`. -- [1290](https://github.com/nginxinc/kubernetes-ingress/pull/1290) Graduate policy resource and accessControl policy to generally available. -- [1225](https://github.com/nginxinc/kubernetes-ingress/pull/1225) Require secrets to have types. -- [1237](https://github.com/nginxinc/kubernetes-ingress/pull/1237) Deprecate support for helm2 clients. -UPGRADE: +### Upgrade - For NGINX, use the 1.10.0 image from our DockerHub: `nginx/nginx-ingress:1.10.0`, `nginx/nginx-ingress:1.10.0-alpine` or `nginx-ingress:1.10.0-ubi` - For NGINX Plus, please build your own image using the 1.10.0 source code. @@ -1367,32 +1301,35 @@ NOTES: - Helm 2 clients are no longer supported due to reaching End of Life: -## NGINX Ingress Controller 1.9.1 +
+ +## 1.9.1 23 November 2020 -CHANGES: +### Dependencies -- Fix deployment of ingressclass resource via helm on some versions of Kubernetes. - Update the base ubi images to 8.3. -- Renew CA cert for egress-mtls example. -- Add imagePullSecretName support to helm chart. -HELM CHART: +### Features -- The version of the Helm chart is now 0.7.1. +- Fix deployment of ingressclass resource via helm on some versions of Kubernetes. +- Renew CA cert for egress-mtls example. +- Add imagePullSecretName support to helm chart. -UPGRADE: +### Upgrade - For NGINX, use the 1.9.1 image from our DockerHub: `nginx/nginx-ingress:1.9.1`, `nginx/nginx-ingress:1.9.1-alpine` or `nginx/nginx-ingress:1.9.1-ubi` - For NGINX Plus, please build your own image using the 1.9.1 source code. - For Helm, use version 0.7.1 of the chart. -## NGINX Ingress Controller 1.9.0 +
+ +## 1.9.0 20 October 2020 -OVERVIEW: +### Overview Release 1.9.0 includes: @@ -1403,30 +1340,21 @@ Release 1.9.0 includes: You will find the complete changelog for release 1.9.0, including bug fixes, improvements, and changes below. -FEATURES FOR POLICY RESOURCE: +### Features - [1180](https://github.com/nginxinc/kubernetes-ingress/pull/1180) Add support for EgressMTLS. - [1166](https://github.com/nginxinc/kubernetes-ingress/pull/1166) Add IngressMTLS policy support. - [1154](https://github.com/nginxinc/kubernetes-ingress/pull/1154) Add JWT policy support. - [1120](https://github.com/nginxinc/kubernetes-ingress/pull/1120) Add RateLimit policy support. - [1058](https://github.com/nginxinc/kubernetes-ingress/pull/1058) Support policies in VS routes and VSR subroutes. - -FEATURES FOR NGINX APP PROTECT: - - [1147](https://github.com/nginxinc/kubernetes-ingress/pull/1147) Add option to specify other log destinations in AppProtect. - [1131](https://github.com/nginxinc/kubernetes-ingress/pull/1131) Update packages and CRDs to AppProtect 2.0. This update includes features such as: [JSON Schema Validation](https://docs.nginx.com/nginx-app-protect/configuration#applying-a-json-schema), [User-Defined URLs](https://docs.nginx.com/nginx-app-protect/configuration/#user-defined-urls) and [User-Defined Parameters](https://docs.nginx.com/nginx-app-protect/configuration/#user-defined-parameters). See the [release notes](https://docs.nginx.com/nginx-app-protect/releases/#release-2-0) for a complete feature list. - [1100](https://github.com/nginxinc/kubernetes-ingress/pull/1100) Add external references to AppProtect. - [1085](https://github.com/nginxinc/kubernetes-ingress/pull/1085) Add installation of threat campaigns package. - -FEATURES: - - [1133](https://github.com/nginxinc/kubernetes-ingress/pull/1133) Add support for IngressClass resources. - [1130](https://github.com/nginxinc/kubernetes-ingress/pull/1130) Add prometheus latency collector. - [1076](https://github.com/nginxinc/kubernetes-ingress/pull/1076) Add prometheus worker process metrics. - [1075](https://github.com/nginxinc/kubernetes-ingress/pull/1075) Add support for NGINX Service Mesh internal routes. - -IMPROVEMENTS: - - [1178](https://github.com/nginxinc/kubernetes-ingress/pull/1178) Resolve host collisions in VirtualServer and Ingresses. - [1158](https://github.com/nginxinc/kubernetes-ingress/pull/1158) Support variables in action proxy headers. - [1137](https://github.com/nginxinc/kubernetes-ingress/pull/1137) Add pod_owner label to metrics when -spire-agent-address is set. @@ -1437,23 +1365,22 @@ IMPROVEMENTS: - [1078](https://github.com/nginxinc/kubernetes-ingress/pull/1078) Remove trailing blank lines in vs/vsr snippets. - Documentation improvements: [1083](https://github.com/nginxinc/kubernetes-ingress/pull/1083), [1092](https://github.com/nginxinc/kubernetes-ingress/pull/1092), [1089](https://github.com/nginxinc/kubernetes-ingress/pull/1089), [1174](https://github.com/nginxinc/kubernetes-ingress/pull/1174), [1175](https://github.com/nginxinc/kubernetes-ingress/pull/1175), [1171](https://github.com/nginxinc/kubernetes-ingress/pull/1171). -BUGFIXES: +### Fixes - [1179](https://github.com/nginxinc/kubernetes-ingress/pull/1179) Fix TransportServers in debian AppProtect image. - [1129](https://github.com/nginxinc/kubernetes-ingress/pull/1129) Support real-ip in default server. - [1110](https://github.com/nginxinc/kubernetes-ingress/pull/1110) Add missing threat campaigns key to AppProtect CRD. -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.7.0 - [1105](https://github.com/nginxinc/kubernetes-ingress/pull/1105) Fix GlobalConfiguration support in helm chart. - Add new parameters to the Chart: `controller.setAsDefaultIngress`, `controller.enableLatencyMetrics`. Added in [1133](https://github.com/nginxinc/kubernetes-ingress/pull/1133) and [1148](https://github.com/nginxinc/kubernetes-ingress/pull/1148). -CHANGES: +### Dependencies - [1182](https://github.com/nginxinc/kubernetes-ingress/pull/1182) Update NGINX version to 1.19.3. -UPGRADE: +### Upgrade - For NGINX, use the 1.9.0 image from our DockerHub: `nginx/nginx-ingress:1.9.0`, `nginx/nginx-ingress:1.9.0-alpine` or `nginx-ingress:1.9.0-ubi` - For NGINX Plus, please build your own image using the 1.9.0 source code. @@ -1461,39 +1388,39 @@ UPGRADE: For Kubernetes >= 1.18, when upgrading using the [manifests](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/), make sure to update the [ClusterRole](https://github.com/nginxinc/kubernetes-ingress/blob/v1.9.0/deployments/rbac/rbac.yaml) and create the [IngressClass resource](https://github.com/nginxinc/kubernetes-ingress/blob/v1.9.0/deployments/common/ingress-class.yaml), which is required for Kubernetes >= 1.18. Otherwise, the Ingress Controller will fail to start. If you run multiple NGINX Ingress Controllers in the cluster, each Ingress Controller must have its own IngressClass resource. As the `-use-ingress-class-only` argument is now ignored (see NOTES), make sure your Ingress resources have the `ingressClassName` field or the `kubernetes.io/ingress.class` annotation set to the name of the IngressClass resource. Otherwise, the Ingress Controller will ignore them. -HELM UPGRADE: +#### Helm Upgrade - If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs will not be upgraded. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to upgrade the CRDs. - For Kubernetes >= 1.18, a dedicated IngressClass resource, which is configured by `controller.ingressClass`, is required per helm release. Ensure `controller.ingressClass` is not set to the name of the IngressClass of other releases or Ingress Controllers. As the `controller.useIngressClassOnly` parameter is now ignored (see NOTES), make sure your Ingress resources have the `ingressClassName` field or the `kubernetes.io/ingress.class` annotation set to the value of `controller.ingressClass`. Otherwise, the Ingress Controller will ignore them. -NOTES: +### Notes - When using Kubernetes >= 1.18, the `-use-ingress-class-only` command-line argument is now ignored, and the Ingress Controller will only process resources that belong to its class. See [IngressClass doc](https://docs.nginx.com/nginx-ingress-controller/installation/running-multiple-ingress-controllers/#ingress-class) for more details. - For Kubernetes >= 1.18, a dedicated IngressClass resource, which is configured by `controller.ingressClass`, is required per helm release. When upgrading or installing releases, ensure `controller.ingressClass` is not set to the name of the IngressClass of other releases or Ingress Controllers. -## NGINX Ingress Controller 1.8.1 +
+ +## 1.8.1 14 August 2020 -CHANGES: +### Dependencies - Update NGINX version to 1.19.2. -HELM CHART: - -- The version of the Helm chart is now 0.6.1. - -UPGRADE: +### Upgrade - For NGINX, use the 1.8.1 image from our DockerHub: `nginx/nginx-ingress:1.8.1`, `nginx/nginx-ingress:1.8.1-alpine` or `nginx/nginx-ingress:1.8.1-ubi` - For NGINX Plus, please build your own image using the 1.8.1 source code. - For Helm, use version 0.6.1 of the chart. -## NGINX Ingress Controller 1.8.0 +
+ +## 1.8.0 22 July 2020 -OVERVIEW: +### Overview Release 1.8.0 includes: @@ -1513,89 +1440,83 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [994](https://github.com/nginxinc/kubernetes-ingress/pull/994): Support Class Field in VS/VSR. - [973](https://github.com/nginxinc/kubernetes-ingress/pull/973): Add status to VirtualServer and VirtualServerRoute. -FEATURES: +### Features - [1035](https://github.com/nginxinc/kubernetes-ingress/pull/1035): Support for App Protect module. - [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029): Add readiness endpoint. - -IMPROVEMENTS: - - [995](https://github.com/nginxinc/kubernetes-ingress/pull/995): Emit event for orphaned VirtualServerRoutes. +- [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029): Add readiness endpoint. The Ingress Controller now exposes a readiness endpoint on port `8081` and the path `/nginx-ready`. The endpoint returns a `200` response after the Ingress Controller finishes the initial configuration of NGINX at the start. The pod template was updated to use that endpoint in a readiness probe. +- [980](https://github.com/nginxinc/kubernetes-ingress/pull/980): Enable leader election by default. - Documentation improvements: [946](https://github.com/nginxinc/kubernetes-ingress/pull/946) thanks to [谭九鼎](https://github.com/imba-tjd), [948](https://github.com/nginxinc/kubernetes-ingress/pull/948), [972](https://github.com/nginxinc/kubernetes-ingress/pull/972), [965](https://github.com/nginxinc/kubernetes-ingress/pull/965). -BUGFIXES: +### Fixes - [1030](https://github.com/nginxinc/kubernetes-ingress/pull/1030): Fix port range validation in cli arguments. - [953](https://github.com/nginxinc/kubernetes-ingress/pull/953): Fix error logging of master/minion ingresses. -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.6.0. - Add new parameters to the Chart: `controller.appprotect.enable`, `controller.globalConfiguration.create`, `controller.globalConfiguration.spec`, `controller.readyStatus.enable`, `controller.readyStatus.port`, `controller.config.annotations`, `controller.reportIngressStatus.annotations`. Added in [1035](https://github.com/nginxinc/kubernetes-ingress/pull/1035), [1034](https://github.com/nginxinc/kubernetes-ingress/pull/1034), [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029), [1003](https://github.com/nginxinc/kubernetes-ingress/pull/1003) thanks to [RubyLangdon](https://github.com/RubyLangdon). - [1047](https://github.com/nginxinc/kubernetes-ingress/pull/1047) and [1009](https://github.com/nginxinc/kubernetes-ingress/pull/1009): Change how Helm manages the custom resource definitions (CRDs) to support installing multiple Ingress Controller releases. **Note**: If you're using the custom resources (`controller.enableCustomResources` is set to `true`), this is a breaking change. See the HELM UPGRADE section below for the upgrade instructions. -CHANGES: +### Dependencies - Update NGINX version to 1.19.1. - Update NGINX Plus to R22. -- [1029](https://github.com/nginxinc/kubernetes-ingress/pull/1029): Add readiness endpoint. The Ingress Controller now exposes a readiness endpoint on port `8081` and the path `/nginx-ready`. The endpoint returns a `200` response after the Ingress Controller finishes the initial configuration of NGINX at the start. The pod template was updated to use that endpoint in a readiness probe. -- [980](https://github.com/nginxinc/kubernetes-ingress/pull/980): Enable leader election by default. -UPGRADE: +### Upgrade - For NGINX, use the 1.8.0 image from our DockerHub: `nginx/nginx-ingress:1.8.0`, `nginx/nginx-ingress:1.8.0-alpine` or `nginx-ingress:1.8.0-ubi` - For NGINX Plus, please build your own image using the 1.8.0 source code. - For Helm, use version 0.6.0 of the chart. -HELM UPGRADE: +#### Helm Upgrade If you're using custom resources like VirtualServer and TransportServer (`controller.enableCustomResources` is set to `true`), after you run the `helm upgrade` command, the CRDs and the corresponding custom resources will be removed from the cluster. Before upgrading, make sure to back up the custom resources. After running the `helm upgrade` command, run `kubectl apply -f deployments/helm-chart/crds` to re-install the CRDs and then restore the custom resources. -NOTES: +### Notes - As part of installing a release, Helm will install the CRDs unless that step is disabled (see the [corresponding doc](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-helm/)). The installed CRDs include the CRDs for all Ingress Controller features, including the ones disabled by default (like App Protect with `aplogconfs.appprotect.f5.com` and `appolicies.appprotect.f5.com` CRDs). -## NGINX Ingress Controller 1.7.2 +
+ +## 1.7.2 23 June 2020 -CHANGES: +### Dependencies - Update NGINX Plus version to R22. -HELM CHART: - -- The version of the Helm chart is now 0.5.2. - -UPGRADE: +### Upgrade - For NGINX, use the 1.7.2 image from our DockerHub: `nginx/nginx-ingress:1.7.2`, `nginx/nginx-ingress:1.7.2-alpine` or `nginx/nginx-ingress:1.7.2-ubi` - For NGINX Plus, please build your own image using the 1.7.2 source code. - For Helm, use version 0.5.2 of the chart. -## NGINX Ingress Controller 1.7.1 +
+ +## 1.7.1 4 June 2020 -CHANGES: +### Dependencies - Update NGINX version to 1.19.0. -HELM CHART: - -- The version of the Helm chart is now 0.5.1. - -UPGRADE: +### Upgrade - For NGINX, use the 1.7.1 image from our DockerHub: `nginx/nginx-ingress:1.7.1`, `nginx/nginx-ingress:1.7.1-alpine` or `nginx/nginx-ingress:1.7.1-ubi` - For NGINX Plus, please build your own image using the 1.7.1 source code. - For Helm, use version 0.5.1 of the chart. -## NGINX Ingress Controller 1.7.0 +
+ +## 1.7.0 30 April 2020 -OVERVIEW: +### Overview Release 1.7.0 includes: @@ -1613,7 +1534,7 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [868](https://github.com/nginxinc/kubernetes-ingress/pull/868): Add OpenAPI CRD schema validation. - [847](https://github.com/nginxinc/kubernetes-ingress/pull/847): Add support for error pages for VS/VSR. -FEATURES: +### Features - [902](https://github.com/nginxinc/kubernetes-ingress/pull/902): Add TransportServer and GlobalConfiguration Resources. - [894](https://github.com/nginxinc/kubernetes-ingress/pull/894): Add Dockerfile for NGINX Open Source for Openshift. @@ -1621,33 +1542,29 @@ FEATURES: - [852](https://github.com/nginxinc/kubernetes-ingress/pull/852): Add default-server-access-log-off to configmap. - [845](https://github.com/nginxinc/kubernetes-ingress/pull/845): Add log-format-escaping and stream-log-format-escaping configmap keys. Thanks to [Alexey Maslov](https://github.com/alxmsl). - [827](https://github.com/nginxinc/kubernetes-ingress/pull/827): Add ingress class label to all Prometheus metrics. - -IMPROVEMENTS: - - [850](https://github.com/nginxinc/kubernetes-ingress/pull/850): Extend redirect URI validation with protocol check in VS/VSR. - [832](https://github.com/nginxinc/kubernetes-ingress/pull/832): Update the examples to run the `nginxdemos/nginx-hello:plain-text` image, that doesn't require root user. - [825](https://github.com/nginxinc/kubernetes-ingress/pull/825): Add multi-stage docker builds. +- [852](https://github.com/nginxinc/kubernetes-ingress/pull/852): Add default-server-access-log-off to configmap. The access logs for the default server are now enabled by default. +- [847](https://github.com/nginxinc/kubernetes-ingress/pull/847): Add support for error pages for VS/VSR. The PR affects how the Ingress Controller generates configuration for VirtualServer and VirtualServerRoutes. See [this comment](https://github.com/nginxinc/kubernetes-ingress/pull/847) for more details. +- [827](https://github.com/nginxinc/kubernetes-ingress/pull/827): Add ingress class label to all Prometheus metrics. Every Prometheus metric exposed by the Ingress Controller now includes the label `class` with the value of the Ingress Controller class (by default `nginx`), +- [825](https://github.com/nginxinc/kubernetes-ingress/pull/825): Add multi-stage docker builds. When building the Ingress Controller image in Docker, we now use a multi-stage docker build. -BUGFIXES: +### Fixes - [828](https://github.com/nginxinc/kubernetes-ingress/pull/828): Fix error messages for actions of the type return. -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.5.0. - Add new parameters to the Chart: `controller.enableTLSPassthrough`, `controller.volumes`, `controller.volumeMounts`, `controller.priorityClassName`. Added in [921](https://github.com/nginxinc/kubernetes-ingress/pull/921), [878](https://github.com/nginxinc/kubernetes-ingress/pull/878), [807](https://github.com/nginxinc/kubernetes-ingress/pull/807) thanks to [Greg Snow](https://github.com/gsnegovskiy). -CHANGES: +### Dependencies - Update NGINX version to 1.17.10. - Update NGINX Plus to R21. - [854](https://github.com/nginxinc/kubernetes-ingress/pull/854): Update the Debian base images for NGINX Plus to `debian:buster-slim`. -- [852](https://github.com/nginxinc/kubernetes-ingress/pull/852): Add default-server-access-log-off to configmap. The access logs for the default server are now enabled by default. -- [847](https://github.com/nginxinc/kubernetes-ingress/pull/847): Add support for error pages for VS/VSR. The PR affects how the Ingress Controller generates configuration for VirtualServer and VirtualServerRoutes. See [this comment](https://github.com/nginxinc/kubernetes-ingress/pull/847) for more details. -- [827](https://github.com/nginxinc/kubernetes-ingress/pull/827): Add ingress class label to all Prometheus metrics. Every Prometheus metric exposed by the Ingress Controller now includes the label `class` with the value of the Ingress Controller class (by default `nginx`), -- [825](https://github.com/nginxinc/kubernetes-ingress/pull/825): Add multi-stage docker builds. When building the Ingress Controller image in Docker, we now use a multi-stage docker build. -UPGRADE: +### Upgrade - For NGINX, use the 1.7.0 image from our DockerHub: `nginx/nginx-ingress:1.7.0`, `nginx/nginx-ingress:1.7.0-alpine` or `nginx-ingress:1.7.0-ubi` - For NGINX Plus, please build your own image using the 1.7.0 source code. @@ -1655,65 +1572,61 @@ UPGRADE: When upgrading using the [manifests](/nginx-ingress-controller/installation/installation-with-manifests/), make sure to deploy the new TransportServer CRD (`common/ts-definition.yaml`), as it is required by the Ingress Controller. Otherwise, you will get error messages in the Ingress Controller logs. -## NGINX Ingress Controller 1.6.3 +
+ +## 1.6.3 6 March 2020 -CHANGES: +### Dependencies - Update NGINX version to 1.17.9. -HELM CHART: - -- The version of the Helm chart is now 0.4.3. - -UPGRADE: +### Upgrade - For NGINX, use the 1.6.3 image from our DockerHub: `nginx/nginx-ingress:1.6.3` or `nginx/nginx-ingress:1.6.3-alpine` - For NGINX Plus, please build your own image using the 1.6.3 source code. - For Helm, use version 0.4.3 of the chart. -## NGINX Ingress Controller 1.6.2 +
+ +## 1.6.2 6 February 2020 -CHANGES: +### Dependencies - Update NGINX version to 1.17.8. -HELM CHART: - -- The version of the Helm chart is now 0.4.2. - -UPGRADE: +### Upgrade - For NGINX, use the 1.6.2 image from our DockerHub: `nginx/nginx-ingress:1.6.2` or `nginx/nginx-ingress:1.6.2-alpine` - For NGINX Plus, please build your own image using the 1.6.2 source code. - For Helm, use version 0.4.2 of the chart. -## NGINX Ingress Controller 1.6.1 +
+ +## 1.6.1 14 January 2020 -CHANGES: +### Dependencies - Update NGINX version to 1.17.7. -HELM CHART: - -- The version of the Helm chart is now 0.4.1. - -UPGRADE: +### Upgrade - For NGINX, use the 1.6.1 image from our DockerHub: `nginx/nginx-ingress:1.6.1` or `nginx/nginx-ingress:1.6.1-alpine` - For NGINX Plus, please build your own image using the 1.6.1 source code. - For Helm, use version 0.4.1 of the chart. -## NGINX Ingress Controller 1.6.0 +
+ +## 1.6.0 19 December 2019 -OVERVIEW: +### Overview Release 1.6.0 includes: @@ -1753,7 +1666,7 @@ FEATURES FOR VIRTUALSERVER AND VIRTUALSERVERROUTE RESOURCES: - [607](https://github.com/nginxinc/kubernetes-ingress/pull/607): Add fail-timeout and max-fails support to vs/vsr. - [596](https://github.com/nginxinc/kubernetes-ingress/pull/596): Add lb-method support in vs and vsr. -FEATURES: +### Features - [750](https://github.com/nginxinc/kubernetes-ingress/pull/750): Add support for health status uri customisation. - [691](https://github.com/nginxinc/kubernetes-ingress/pull/691): Helper Functions for custom annotations. @@ -1762,14 +1675,17 @@ FEATURES: - [616](https://github.com/nginxinc/kubernetes-ingress/pull/616): Add proxy-send-timeout to configmap key and annotation. - [615](https://github.com/nginxinc/kubernetes-ingress/pull/615): Add support for Opentracing. - [614](https://github.com/nginxinc/kubernetes-ingress/pull/614): Add max-conns annotation. Thanks to [Victor Regalado](https://github.com/vrrs). - -IMPROVEMENTS: - - [678](https://github.com/nginxinc/kubernetes-ingress/pull/678): Increase defaults for server-names-hash-max-size and servers-names-hash-bucket-size ConfigMap keys. - [694](https://github.com/nginxinc/kubernetes-ingress/pull/694): Reject VS/VSR resources with enabled plus features for OSS. +- [799](https://github.com/nginxinc/kubernetes-ingress/pull/779): Enable CRDs by default. VirtualServer and VirtualServerRoute resources are now enabled by default. +- [772](https://github.com/nginxinc/kubernetes-ingress/pull/772): Update VS/VSR version from v1alpha1 to v1. Make sure to update the `apiVersion` of your VirtualServer and VirtualServerRoute resources. +- [748](https://github.com/nginxinc/kubernetes-ingress/pull/748): Add TLS redirect support in VirtualServer. The `redirect-to-https` and `ssl-redirect` ConfigMap keys no longer have any effect on generated configs for VirtualServer resources. +- [745](https://github.com/nginxinc/kubernetes-ingress/pull/745): Improve routing rules. Update the spec of VirtualServer and VirtualServerRoute accordingly. See YAML examples of the changes [here](https://github.com/nginxinc/kubernetes-ingress/pull/745). +- [710](https://github.com/nginxinc/kubernetes-ingress/pull/710): Run IC as non-root. Make sure to use the updated manifests to install/upgrade the Ingress Controller. +- [603](https://github.com/nginxinc/kubernetes-ingress/pull/603): Update apiVersion in Deployments and DaemonSets to apps/v1. - Documentation improvements: [713](https://github.com/nginxinc/kubernetes-ingress/pull/713) thanks to [Matthew Wahner](https://github.com/mattwahner). -BUGFIXES: +### Fixes - [788](https://github.com/nginxinc/kubernetes-ingress/pull/788): Fix VSR updates when namespace is set implicitly. - [736](https://github.com/nginxinc/kubernetes-ingress/pull/736): Init Ingress labeled metrics on start. @@ -1777,31 +1693,24 @@ BUGFIXES: - [664](https://github.com/nginxinc/kubernetes-ingress/pull/664): Fix reporting events for Ingress minions. - [632](https://github.com/nginxinc/kubernetes-ingress/pull/632): Fix hsts support when not using SSL. Thanks to [Martín Fernández](https://github.com/bilby91). -HELM CHART: +### Helm Chart -- The version of the helm chart is now 0.4.0. - Add new parameters to the Chart: `controller.healthCheckURI`, `controller.resources`, `controller.logLevel`, `controller.customPorts`, `controller.service.customPorts`. Added in [750](https://github.com/nginxinc/kubernetes-ingress/pull/750), [636](https://github.com/nginxinc/kubernetes-ingress/pull/636) thanks to [Guilherme Oki](https://github.com/guilhermeoki), [600](https://github.com/nginxinc/kubernetes-ingress/pull/600), [581](https://github.com/nginxinc/kubernetes-ingress/pull/581) thanks to [Alex Meijer](https://github.com/ameijer-corsha). - [722](https://github.com/nginxinc/kubernetes-ingress/pull/722): Fix trailing leader election cm when using helm. This change might lead to a failed upgrade. See the helm upgrade instruction below. - [573](https://github.com/nginxinc/kubernetes-ingress/pull/573): Use Controller name value for app selectors. -CHANGES: +### Dependencies - Update NGINX versions to 1.17.6. - Update NGINX Plus version to R20. -- [799](https://github.com/nginxinc/kubernetes-ingress/pull/779): Enable CRDs by default. VirtualServer and VirtualServerRoute resources are now enabled by default. -- [772](https://github.com/nginxinc/kubernetes-ingress/pull/772): Update VS/VSR version from v1alpha1 to v1. Make sure to update the `apiVersion` of your VirtualServer and VirtualServerRoute resources. -- [748](https://github.com/nginxinc/kubernetes-ingress/pull/748): Add TLS redirect support in VirtualServer. The `redirect-to-https` and `ssl-redirect` ConfigMap keys no longer have any effect on generated configs for VirtualServer resources. -- [745](https://github.com/nginxinc/kubernetes-ingress/pull/745): Improve routing rules. Update the spec of VirtualServer and VirtualServerRoute accordingly. See YAML examples of the changes [here](https://github.com/nginxinc/kubernetes-ingress/pull/745). -- [710](https://github.com/nginxinc/kubernetes-ingress/pull/710): Run IC as non-root. Make sure to use the updated manifests to install/upgrade the Ingress Controller. -- [603](https://github.com/nginxinc/kubernetes-ingress/pull/603): Update apiVersion in Deployments and DaemonSets to apps/v1. -UPGRADE: +### Upgrade - For NGINX, use the 1.6.0 image from our DockerHub: `nginx/nginx-ingress:1.6.0` or `nginx/nginx-ingress:1.6.0-alpine` - For NGINX Plus, please build your own image using the 1.6.0 source code. - For Helm, use version 0.4.0 of the chart. -HELM UPGRADE: +#### Helm upgrade If leader election (the `controller.reportIngressStatus.enableLeaderElection` parameter) is enabled, when upgrading to the new version of the Helm chart: @@ -1810,6 +1719,8 @@ If leader election (the `controller.reportIngressStatus.enableLeaderElection` pa Otherwise, the helm upgrade will not succeed. +
+ ## Previous Releases -To see the previous releases, see the [Releases page]({{< relref "releases.md" >}}) on the Ingress Controller GitHub repo. +To see the previous releases, see the [Releases page](https://github.com/nginxinc/kubernetes-ingress/releases) on the Ingress Controller GitHub repo. \ No newline at end of file From 10a3618a685276689ce89d848922156cb5e9725d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 13 Nov 2023 15:18:44 +0000 Subject: [PATCH 15/17] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- docs/content/releases.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/content/releases.md b/docs/content/releases.md index 230355743bc..d10042ad634 100644 --- a/docs/content/releases.md +++ b/docs/content/releases.md @@ -1723,4 +1723,4 @@ Otherwise, the helm upgrade will not succeed. ## Previous Releases -To see the previous releases, see the [Releases page](https://github.com/nginxinc/kubernetes-ingress/releases) on the Ingress Controller GitHub repo. \ No newline at end of file +To see the previous releases, see the [Releases page](https://github.com/nginxinc/kubernetes-ingress/releases) on the Ingress Controller GitHub repo. From 37741bd43dffbf86eeff4effcc3485a6ffdc6da9 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Tue, 14 Nov 2023 11:26:20 +0000 Subject: [PATCH 16/17] Add missing parameters to Helm table --- .../installing-nic/installation-with-helm.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index 8fd0d30b1f5..83fa67f60c1 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -335,8 +335,9 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **controller.extraContainers** | Extra (eg. sidecar) containers for the Ingress Controller pods. | [] | | **controller.resources** | The resources of the Ingress Controller pods. | requests: cpu=100m,memory=128Mi | | **controller.replicaCount** | The number of replicas of the Ingress Controller deployment. | 1 | -| **controller.ingressClass** | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of Kubernetes. | nginx | -| **controller.setAsDefaultIngress** | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. | false | +| **controller.ingressClass.name** | A class of the Ingress Controller. An IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress Controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of Kubernetes. | nginx | +| **controller.ingressClass.create** | Creates a new IngressClass object with the name `controller.ingressClass.name`. Set to `false` to use an existing ingressClass created using `kubectl` with the same name. If you use `helm upgrade`, do not change the values from the previous release as helm will delete IngressClass objects managed by helm. If you are upgrading from a release earlier than 3.3.2, do not set the value to false. | true | +| **controller.setAsDefaultIngress** | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass.name`. | false | | **controller.watchNamespace** | Comma separated list of namespaces the Ingress Controller should watch for resources. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchNamespace="default\,nginx-ingress"`. | "" | | **controller.watchNamespaceLabel** | Configures the Ingress Controller to watch only those namespaces with label foo=bar. By default the Ingress Controller watches all namespaces. Mutually exclusive with `controller.watchNamespace`. | "" | | **controller.watchSecretNamespace** | Comma separated list of namespaces the Ingress Controller should watch for resources of type Secret. If this arg is not configured, the Ingress Controller watches the same namespaces for all resources. See `controller.watchNamespace` and `controller.watchNamespaceLabel`. Please note that if configuring multiple namespaces using the Helm cli `--set` option, the string needs to wrapped in double quotes and the commas escaped using a backslash - e.g. `--set controller.watchSecretNamespace="default\,nginx-ingress"`. | "" | @@ -363,6 +364,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **controller.service.extraLabels** | The extra labels of the service. | {} | | **controller.service.loadBalancerIP** | The static IP address for the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | "" | | **controller.service.externalIPs** | The list of external IPs for the Ingress Controller service. | [] | +| **controller.service.clusterIP** | The clusterIP for the Ingress Controller service, autoassigned if not specified. | "" | | **controller.service.loadBalancerSourceRanges** | The IP ranges (CIDR) that are allowed to access the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | [] | | **controller.service.name** | The name of the service. | Autogenerated | | **controller.service.customPorts** | A list of custom ports to expose through the Ingress Controller service. Follows the conventional Kubernetes yaml syntax for service ports. | [] | @@ -419,6 +421,12 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **prometheus.port** | Configures the port to scrape the metrics. | 9113 | | **prometheus.scheme** | Configures the HTTP scheme to use for connections to the Prometheus endpoint. | http | | **prometheus.secret** | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Prometheus endpoint with TLS connections. | "" | +| **prometheus.service.create** | Create a Headless service to expose prometheus metrics. Requires `prometheus.create`. | false | +| **prometheus.service.labels** | Kubernetes object labels to attach to the service object. | {service: "nginx-ingress-prometheus-service"} | +| **prometheus.serviceMonitor.create** | Create a ServiceMonitor custom resource. Requires ServiceMonitor CRD to be installed. For the latest CRD, check the latest release on the [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) GitHub repo under `example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml` | false | +| **prometheus.serviceMonitor.labels** | Kubernetes object labels to attach to the serviceMonitor object. | {} | +| **prometheus.serviceMonitor.selectorMatchLabels** | A set of labels to allow the selection of endpoints for the ServiceMonitor. | {service: "nginx-ingress-prometheus-service"} | +| **prometheus.serviceMonitor.endpoints** | A list of endpoints allowed as part of this ServiceMonitor. | [port: prometheus] | | **serviceInsight.create** | Expose NGINX Plus Service Insight endpoint. | false | | **serviceInsight.port** | Configures the port to expose endpoints. | 9114 | | **serviceInsight.scheme** | Configures the HTTP scheme to use for connections to the Service Insight endpoint. | http | @@ -428,8 +436,6 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **nginxServiceMesh.enableEgress** | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false | {{}} - - ## Notes - The values-icp.yaml file is used for deploying the Ingress Controller on IBM Cloud Private. See the [blog post](https://www.nginx.com/blog/nginx-ingress-controller-ibm-cloud-private/) for more details. From 68b409e88138f4c7aa2fb6bf4856c17912521105 Mon Sep 17 00:00:00 2001 From: Alan Dooley Date: Tue, 14 Nov 2023 12:19:37 +0000 Subject: [PATCH 17/17] Fix Helm installation table, delete redundant images --- .../installing-nic/installation-with-helm.md | 2 ++ docs/content/overview/control-loop.png | Bin 5633 -> 0 bytes docs/content/overview/controller-sync.png | Bin 34320 -> 0 bytes docs/content/overview/design.md | 12 ++++++------ docs/content/overview/ic-high-level.png | Bin 99912 -> 0 bytes docs/content/overview/ic-pod.png | Bin 54934 -> 0 bytes docs/content/overview/ic-process-components.png | Bin 51942 -> 0 bytes docs/content/overview/ic-process.png | Bin 24035 -> 0 bytes 8 files changed, 8 insertions(+), 6 deletions(-) delete mode 100644 docs/content/overview/control-loop.png delete mode 100644 docs/content/overview/controller-sync.png delete mode 100644 docs/content/overview/ic-high-level.png delete mode 100644 docs/content/overview/ic-pod.png delete mode 100644 docs/content/overview/ic-process-components.png delete mode 100644 docs/content/overview/ic-process.png diff --git a/docs/content/installation/installing-nic/installation-with-helm.md b/docs/content/installation/installing-nic/installation-with-helm.md index 83fa67f60c1..85f213747ae 100644 --- a/docs/content/installation/installing-nic/installation-with-helm.md +++ b/docs/content/installation/installing-nic/installation-with-helm.md @@ -295,6 +295,7 @@ See [running multiple Ingress Controllers]({{< relref "installation/running-mult The following tables lists the configurable parameters of the NGINX Ingress Controller chart and their default values. +{{< table >}} {{}} |Parameter | Description | Default | | --- | --- | --- | @@ -435,6 +436,7 @@ The following tables lists the configurable parameters of the NGINX Ingress Cont | **nginxServiceMesh.enable** | Enable integration with NGINX Service Mesh. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/) for more details. Requires `controller.nginxplus`. | false | | **nginxServiceMesh.enableEgress** | Enable NGINX Service Mesh workloads to route egress traffic through the Ingress Controller. See the NGINX Service Mesh [docs](https://docs.nginx.com/nginx-service-mesh/tutorials/kic/deploy-with-kic/#enabling-egress) for more details. Requires `nginxServiceMesh.enable`. | false | {{}} +{{< /table >}} ## Notes diff --git a/docs/content/overview/control-loop.png b/docs/content/overview/control-loop.png deleted file mode 100644 index c70944cb4d45250ed14591efb4c151670a59dfc0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5633 zcmds5XHXN)wvQo5vjNgUL=cevr-hF64n}$jRZvRkF%T(6P~;y8D!qsz9i$UV5QqX| zK!S7#0uhKv?>)f9_deWtAMS@YZ{EziZ)bLP_ILK2J!kjK*>g7jj=8}_Ms7v`0C3UB zQ1>nXK!XPW&PmYIoFS^rV&P{4=)RVj764G4$V7FcJ)2+mFuZF903d|{fXA@_z~LG6 z_%{FmmjM74T>$` zEN|_t?H=NAxMuw4*@czff#Kh4TfTmN!OvdZkFa_Ct~UMqTE@Vp|4T@CoguQ`Ft(&6 zFxA4w2mZ2SCb?$;Ro>|plcZ;1?SgpvC=d6dl?21&ruD6Pr|Fous)Uy52NmeK#@%*| z(uC#chSeHCeAR5=>JO_7)U7|6^75!uYV5PP4#KzEhNg~=4jp!#%Cn<5=Wb%43wTb;XLvSY2sE@00RT9&{`7N3 z@R;N?kS^57OplIoo{kmBqjBKa2LP}x8tH0Tg^^aW86FLgaF5H%?sx}@HW!rR$^}Uw z+UdZWVQ9+rMLuHS+VN={rQ+5FcASZh-OHWUK96P=Z9>=Kl)}xR`*df*|JNG+7Q*Q* z03gu?71&;z)P~SQBus^DCbXB?)|xc*0Cur#`@|ekw!OXls%CHA_jL z#|NCKwd9USPyd74yh7(WI-!wxR2cbC=$xyDJ~0)gpRcHX7boQ$OS<}pl>!?Ug|J$KmBCN0FZ5`cVe;SsA56wFbAti>blM6UMZlF zBih8Q(m;^;7t&%@Ua#UVcTBZfQTs#B*6S?U*NfZB2>~^)9wGIXAohH=W7wF**J8#9 z1cI{ecoP49yII267&h){VGMZ5=Rh_f&?C7e0}oasb3-K;1sP{t4Nk>`{n;5Q243@m z<>ghrBS|cPdvR?0dPjYHJ_02}PF`lpFoAPr{u+bvh+j z_~{03n9po(*n=s{f%XSFTD`AC)R1hcgB)bMQ#GV zV&>eQZMN1m)I|pjco12`=5rk~#TJ7$XbO&K;8gV%OTIt@N$6ThR*mpYfE`^kA|Ti* zJh;!MNo~Fy(QRw;FFcj39+xr=7%``OhyR*?k~b^%m=%-gbpLeMQaK2UCQFO3jUKeE z@OnpEoEyO?rfVqi`A{Qv_Zje{@Cf0gwK43uiQe4JQMlfdO3@g5g0YTXRLpi8`FWS@ zaXM|$Pq6YS{EwL6Yt$hujpIdNj0rIvR78XNbhAw`P~{4z(-ZtHE!RK>U=ipHqt9jm zolmweAZb z4V8y{3+|Z25yABXrqNfzAC>x~YU2kYlgl_7gJ)ETOG+glbjb5KqKk^bDAKEi72X8zjH1D07@IF-P?qsL_<|+(Mp}~X$ zH7((*xX%|$N8}u41-i|i2M+`J#iF5T9H-~Hk8RB_h6pK5Pa2Y6Cr`K-SMTNcVAl6e zSUlT-01xR@Y>F`GxFQjjgWoiXT)@?{&iz=B$Qly@pBdwvrUsNIgu#J%!vxgBt_}Qp|D8mTC0Ui|eXzL0 zSMLXo4gZ)%R$$^)%#`f+K`b_W&=v5(8Xk{z47F@kv7RrRTk*+gxPQ z@`|%qX!YW3W3Kh89vsWI;O;2zi5bmvmR$VJ7xW|EuiF%aQ6Wr;zV4FucUa)culLEY z;@te!36BhvIN#6>tu6s3#(JZ*`)lS!#j8E`2r3s;g9hxLCWMWdv{qu&^sSr9Cyg$} zLh7KvJy@SFC{^8Onlt>MT{MfHs)Np=_v*mD&_(jQz* zdH34xq3R+$+|Oi<9w`Tm%UfvKw4ei*Zx0up>EQ>dH(h-3*}S2|G0%Y$JZDpzi$*=@ z3@Ue@xp!|v>+!QsG>iaSiXukpw z3@@)63h%lx;)wDuEfT9PDwF#CTMTV_saR7`yk3CaLR!(1oaoYlfKqcosrtf}q@K4f zMUTjueu;jxtyE!+Ta%hnlu8d&^Q=5r)G{j}wL;xGl`#3*qsCc&Kt1sK{hspr9I)c> zJJob%yB5j6lUg>v?iZ%C9uDRBWPHXL_hSg%_5K<9I8W|Szn**Zcww}G3VJ19siq$u z%HFW^DJBaTgW@?e!J|I{kgUi)fPO* znptHT?TJ*B6wCf*^Kys?&?L8OYX-)=KgxDZ)v2_sJ7F1(gvv38xyoM_Hb|uPcf1d* zZD`Rgn0{6|$3H(fuFgjpPW2udw7<1AR&Q!#PPeiO`xJdDJPg#K+!S&9!$j_?mcHL4_(p&bcwAa3kSV-HiLw4B5V~3iu=|9Zd8nMxd zS$_eVlzZsC+qV0-+uS{r)4RGL8f~W5%tuP~R!SL+$IIDM(fGk=Q-c7IxkJsXgBzqJ zOmGDri&|EBRJ!@1D0q_}gsSjn^{e@1mzbp`bC-fqK;}EA#V|q>CAZ!@`IyxeX@e!aQdK3Z{3;zeo6qibmCr_uDsA%!VWg z+s4wEdG1u>Xu3pnwj}k|fM7SRBN|h}f zRQ!D}OwYC1X1I>->92|YAX8Fn8@rOnnVYmtFi1T5arxLqO0buCexe$WAKkDzNKEd|d;RpRBliINC#Y$L)*DrjMM7?3%EEzMIlcXQy9RL=$8I+nIL0@^1ItbG9%k{osz|r%(<9WEHSh#^Bd}Pi zeS79pV+J@$tZA}%M+IjTs2+Rfpq#?Y^IQN0c|y)(}UNBzn8i-Wh}Nlf{q zt1aT*1jT#w|fKezyKQ45ZPS2x_U0jC%s0~w`(YAdX%>63sI_MH@kKuWrJ)F;Ya-Oc5^28G z=tJ*akwjKf`^j<-BI1o&Y}ojkbS=w@Gg+aw)sgoGe7H!JH-K<)Qg(qS+jlo=-msvu zWk=aBl(M#}gq}r8HgVt-$E|sc*{r6@;((tU`Q*(^*t>GkqP9B4I`kdEiFyGu3A{_^ zQ2~S+FxM1d%k*cB{YMyt=#l*4vBcRUBRpjskLRjwIvsyOm?56gJR!saFiZJE$8XMR zZP4Tjey`i)7vtn1|G!gl*g&>Z<*xGv${fc+Bk!7?{K-Dw6g;%l)XLZs<#>0}@Qm$G zBDyS#wUvpOdLn8I0t!|c{Qt~f{|AY9b_mMl(JRu4{z4W$Ad+?86N^ke9ohvOPlKYY z%QbqP$Xh;3VE`L+S4twA3n3y8~e4=w^Vl}E9y z3?IKf@Hb4(Vy1M{PL*UgVzyMwZB%KTh1Qeiiz);avb{#+nE&jBJlhKcHNd6MvhB4o z(MD~~jUSAcco2zl4&wsd3GuQtD8$1{;ueYEmqjVV6)sR|89et_UDvm_{~fE_e-`7} z0{>B_2RoE&z*q1#mL)c2iO|s-``sQHQEa1^{1oCgdDKFn31snL5L7*M1^+PQ!-c}j z=jqgJPGo7#yZYm)EM-Oa+h{(d)LN#5ix^Z~9D686as_Z?%g~tj_E#@CfLxRdPocDN zDMeiX-(TKOcf|k2XT^UL9=mkH4^lmEZX3fxI%B;Vs%IB^&n?tb#Ut4BYyim0$o?fM zqbMl@w)#t1MMhRd7A!6!qaq_S6R`#QFAT7Nd){8*|C%95z@+hyaz=XQx)s{4PyPYK Cq%Q~n diff --git a/docs/content/overview/controller-sync.png b/docs/content/overview/controller-sync.png deleted file mode 100644 index a965a211b1da9546fa13e430af6fb00ed7d43f75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34320 zcmeFZXH=6}7buK6ic)nHDWOh>E#0Z2gH9!&& zkWPq5Cy;=lh7urz7RncA#@90Q{krSkb<6vMb;y%E`|NYh?&UnYab1^_O@NJsg@yC# zl}k5SSXhxPEPJl(XFg!DLd`EThu<777+hdsDT-p>wAsu2e9ZRBO#>Ddm?#U&!zU~( z8_Y)!r&w6LWLa1!)+{Wl@hmL750a~l)R|xGzNe>qiDf4=s=bE!>A-_4W*#god`Vyb zc3t%fe$9Ny>Uq`RGVAz0*2DV_-@1Nuoe7QQ>ZJ>}d=FRHFg*XE=zzwDPnx@@A`kVRQ*1mj_GzT0c`Se91U;kp zaZ1rq`@52suUsy_uMn5fPvmD#$?~wP?#kh>|FDeseL(K|I(~d;w1+u&&gSLe;ps(`+o>Pc$`3)ibjE8T-P30Ixp(+BKP8shZ0r>NLl0UWc=Ro~U-dEDC($QfE;&|1`jtGSM zY1@a7`vWwv*dXWnX^3kxwesQ2`o^bNkB`TWV`V8mv6J&I^r`JRedUEow|?aaOdzBe zKzLuMM(p1=*tWp}3t_O)WmUTAPc0rDW-RYLA)ib4a3tOJ-w_ z{yx-)!-XKSChjn_9OH*B@+zR*bJk4HZ29 zbkLjzaPceJQ#`wuiAc<)0*7pmK1_xBWY_7aG|w7?E$f#0U~ZeDN?D2>trlsu z_3q=$S}l@fMs#bc9ovlZT2*yf3pKMT2LKkK#rK9Wr+gJ-1=(sQdce{Sk)*O+KW|{+ z)RLL1Go{sHT017;xqWrTq;gvhwk->1@$6Q@dp`wd%V;6A0UZjzSuelw)}ml0Lq=!$M(=IB^PslS=3!aFNHk2S=undW1$&(^iAXX z12V%i@2snF#p(H7&J}H9B{n6Mvr8?Bs^k;;kR{dKe;#3p{nfTh=IJ9Dlmr0bpmJOr zxP6NhBCx;n6x>!j5Wm+P-xpe*d~#pm#v@hDHQ5m9v_yvcl5Lz}zj~r}oMEj7ArNS5 zskC2E2cEm68XekkKsY2?2BiR7YMfAGRBSy!7a2K%0)koi`j`Z&13mE{0wt2hwIOFo zg^cY_;|g;Ynmw@ddD0?$bpN(CgF8_kR_^q-%D9h)Z?1;|KrQu-^-a%%z+R(1ee!FK z;ACe_#br;-+UQ?kb)f?P0c&Z5AnN)X`(9d%ve4WjzB+HY`l!HiBD{U`)Q#;YIFGIiDOU?d221gt>%Jm|-Ux_6r*F8JG9^7<0LD|Et~ZD>t0(56n&3vi-J=MFX^y^w z4c{LATms6Ll*CT31=7bOne6J@qEe8&noC13swC2THI)mK4vb04`8FV_CnOjC2v;X( zIG_WHn=CXN0DFhpk$(2pjUFC=Zo}}0dN}0i2jbW<>Bj+n>gjyB+O*jK>4C%e*R@D|6V+loCNp&+o82JQ zjRFAnpv`;eB_?HX&LrUFv0LldUDs`H*7<2&<`AA2ZqzX_Thh^Za2xSpC#rhF$W+S? zU(_aM#hk0X|GnSblGAWv-Oh)wo^jfBL`Q5E{_akji*x<0>`g=oDxU%W9FIA^S~UJV zmN0ak)`@It+#scNhl6>l1#P=3PEKf+VvQ6Q9=^`xsJYI_bh2uo;y!uX`WyNo7NstG zfT9@}C^Co(k~W~XS)qC~w(r7c%8KmvVY)k)2B{Z?;*g{E;FgMVRFI;}-$gv0^JZJUJaXlz{l zZR__oxyG%WGh0ZNB!~GGjU%EZLi}xpaWOL!IHbht$ReYvQ-AdNBBKDmrZW1ts|~;CG9_gV_4C0%xr*rPbTl zW90mbb{07Fws~Imf=5J#Q=;}mZ?pE+EO2%w{>al zc1Uqm7M#*>lFD(qm82w5%^ahUTf_HNd}Cvjwa*%ob2i)7;1H1HBvQ>NJ*6 zaYkKKYbmX<=}FBfqe|n0m8w7qOPv@D8ktdO_y4qUI>^bR%hB=D z^f0LhsjU~t8yF!Saw}a_NK|w=sLp@2!k?71w!Cc0b;`6D)7T@R-ysFxUh&lTs`$M* zC{QE2}usSTej$l6n%$oJNv z`w*HX#IqjTC46+bo}o|+$!FUKVIt-|)zcmKLvuGC)K?1%R>~}zhGn+MM8Hz(!=!ul z1a#oWfkCRePcWfX4Qujks(%u3d&+d>9bDh6Gz3UtB6b7~C)srOeOaBay`Ro{uNZgJ z@YoJu`2eC|Yxo(L5(V3}B%h{zsLE$B5xRypimRFKR&#o6wY51Lv2KxpaT7XQnHgy4 zRkNuxqxFh7x|B0N8EQiJu3t%!+qS@svJq!YGoSzr(kQOFy92m?$wq{LY$}-E>RVLh zzMES6?!*}OuD~O$_eOZLo;6MH>RmKXuYnLF>deZUS&|PQy=)CjTg-dTs@I&-P1IyjcD!1 zaGzo>=*#QCw=gm&N!p1v)vrRE*lyE}uCWJ9Ht}dddgpYog@oW@^Jx%HO}NG<28i|D zejoRR-t~dx%GQ~^T$AS#tKOT?vX;JGpD`>ji8&4;Oyui2AlH&QajXd8EkiFYb>9)f zhfd^VC)}-=a(k2bocFb^f$qny(GA1=u8&ZzyO|&71hj0!c)iUHrEjKK0n1akuZ@f3 zS?Q@sV&X_;bj_$*tKa4uCwOQ;`qH(9w0aK2iA_rj%Zi*ZheAqzO0&hw%@>|ClcKGR zz_iv-E3lE_C>JDg+#R@$yS1c)(~yUkwND(Xa;WlZY;#Mxg_O{4!gW`rG?Fit-zbb- zDXL~uyJ@^(e>jyJ!CmqYeXvuWy^*||JPilw7pD54Y|AG$5d!q~sKyN!@52s02x6bm z<6=0;5Zqm=C8H`tIcS>#nhD*c&O8+pjGsLTe(F%W>N{@<)I2j43&jg?kt!Der!m%| zn?hXFB(uOeL1Jo|D_dH#t_vlOD^io=EH^DPawffSa#eJ*Qr)MoOdWoC>cyz8*l=O1 zBuz`Jo;PqhNj=Y}r&ZD=Kg}nS^)grND)|7rwiC`~6uViFzlw_g!*QAfh~o*Bc(mBz z!!>M99nDDN(l>woVcjZH1~nm;;o3p@8+BhIz~kX!87r zl3pnsL1mMt3;b@TH{$&*LMR>`3K9_ZQvJ)5RH*uD$N`UwCV}4EK|%AeUh3>RcQ(X; zGiB%5#`W>EuwdR4a+P12$k^uACgf1C(6*ii2Q3hg%I%yvu1fSK1s4ys8*cSh1RME; zvff*sbu^{4yFx$d zHJqI0@aabl3Erskf0K6DT8`p{Rl;4m9Dhui`gEb&w_Z!9QZaS6E$Cd5rA@LiG>xV& zy{;y#7Tq~hfOGcbJrVU8$YY>^a#{CNX&(F7)wYOvuN@!phkwtOb`jyCN5F~HLYk9# zWJ-P5lL~qM-bKuDVE;XAv-0SB(m`Tk{}H#kB4IdNxXtytUCMR2cg8U1mCW#>sPzJB%T6Y~Q>^Y}sg zd3*)r15P{LaF`&=?K=Y)bnfPz2{xitwHrQuQKBW25rMBhc;2*7_IeIVE(CWEQ*f-@ zh!&z7xIoh6H=1m!bPK*gPHRmFmV$a51hey!F9}#BOk_X1NYCz{sJFJ|=4QDccr=`t zh9I^*Ljv9P5o24y4I_sN4!w5lgFBLz1wE{Si(}Kywi%ekKXZ5JE^EmXxf2s2dRWywTe}1M8dV|lqB^+dEsj*ChN6GKmmj|Bsa+}Kr@NAjJ99eQM}kW zX={QED9g8^|IYd>lbXjtD@UtyG}er^n*8au9G|NN>V1p zO_BkH&Fc*L_(#3Kr2t<slfx$igN~dy^1(%WOh>b9 z+bh4IcZ4JtUHDYpq8Evxr!(!;fj}V({T!9@wy7uJQCHYrW@t&W&`kOgT4UFc`|*wB)_Lnzn|XHH8y$`I8A(1*Gi#c9P1N70GrcKW#KcSVdIcZ- zH7{5sJW-!Vu(#g+FhD7xIG0NYXE6YeVN6@7XCjP57|7ukQEHD;oGQ@UGGcI;1XeX5 zL9aLGw*rzDq%plwVT<5L486v!n<&kaqC%_8Rjbv7h!F+gI}QWKmHG0Jl4+iO%a$dW zg@PxtCynCQUhORTHpy!1rW)9MW`n$?)!*~g><8CcU<1!brt#kg+L63`;<-<%KOMl^ zT+-oBotnS@=%o_1hP-4cwOZnGn42=; z8xiB{aj=^mds;HvP4c*(4lR|$tgGT`J3r7;leT)jV$Q0ftcqU8F5)ZfnA;!n=k||= z5(%zJNq-C~AcWJv=>L1_rhJgB^}A==1L3QxVMANrZAAyuzX z^37`ExKgIK$%)n1QVO3J1kG)>^;d2N7CkU6YiD!X^mi{0S-7#!P*YJTVh}HJH)3W} z+J@b1wX18D#EpSUwM7fL6MG?AcQNDR~#oRI$AW#N+F;-VD5?HeKFs>$_zA5%eV$T%T-VdlqsVo=Wp$%RB>y7)% zbx3BU8P7Vk6+d}SO$#h{MMoxF0G7TQ@Q@hg7g<8AX6eShwX@ zihHLba!e8*sh5rm$~`m^5;&l7yimyACS?Zn%GkYyI=@PjQufgu!=5+ zb$5$)n&qoZ7OWBdn!<}e1C6qjv2;V3tHuwBNLU7cvo@&X&>lXorB7LL@6Miw-ok$7 z7)zI${9{+Rs^3(X-sdB4fH{^<`a*qXZU3lZa z=>2#1{!4rRWsU#61q%q_G?~d5?)|t-6x^u(Fu43(B=#ZXaQ<8n2ivq^^Q2t0vT{ID zwak0v@4|3{?i*#q?fSV!j&c^?STen3{b_Y?Z{~M_T*E^4Ty#^QfVZ@vLTO5bI*vG< ztp|O?*I!^-%FFOaWz*k9v%H^9&9FWPOW@Oh25rVgtT z%e>Nj1I*eSz@$FCVcBQwg}winKt%3+LI+$SGBDjU24n>k4kkp>ZpGUxQM}-(2CkJv zWM#GaeQjiTbfF6RbNOBqixm#ng=rPM7*WXw?H=qi+VNhHZvWoMzc7<`%Ha23PJw+a zlP$(*rTrPWe=T;Z{hYBSr>7hxjD9415L9}a6;eyQMmXAjuRBG{kpn$ppSZpw=V@rw zfD|gu*+CGGq*K}R48H57>hozOwGl%d7U&(c7t|%0r?2gLn2}Whw~hwwCA;Iig2~-E zCcMW70IPC}UQqOsF(@C(o2DH^2NeP6l%7F|>reQJM{JnApmnQG9=9-##h;EkL*Vf1 ze`>3x=xnJ4))@|UmOsR;Oa5GO2yS5-K4>~WX5-rv61A5nF8lfeXE2~rwl`UA&T3wl#- zcZ93eu!Gq>MnxF+Tx3$D47ZP9a(!TAk!(?0F__vmuK|hjBX2NQJz!8EZ=tAMYRH>z zj{(S!i>~6naEDjn^i!3Rj66b133$Jf5?7Fi6QRx=8s(r8JwGWuOF?JS><$j_2;=Ll zXJgaMN4fJTo&3?RKJE*N6@`T7A4_U*5L2f3tizs*$A(IXlJ|BxJ=RxK@&*re;raw zq^4XvayWI0vD>BVyOosou{(y)Wg2wRF309f@$_Z2D6_>eXCF=Kaz~-nolhPgj=cTp z*kvhVqm&GpH1Bs`$|vFmH@XUm?vuHj7x`85NRds-g>#LWKhs^{?|ZPBi1>|(aWv47 zpYB~fK`BJs@%fck|6jA<&ZSs7H*c+O_Kxz{X=KL{O+K&&Ykn87s(u&~C}=`dB8ioX z7u2yTm8;C3S_;_Sn{K=~l%%yDLolqF^%zL5p+*e-az0~=*Re5f%HW8z(sK$9SxO0I z$_kn2b0UB;+B79$H}h4d)UJdtqOoaRdVOrep$~ta*AQy=9J18tQvbXburij?TL>uj z>T=igAML6S-f)Aqu75^O*PfkYR6qL1#L`%V9KtF-pl`8zqH(NVe`v};Pr}DxsZ78; zDb@s{iJcGXH-GSmhzykkOcc5K9|_f{^0Sr*I-7_I{FIgdGvQnprhXd+g&*drTpu#ULlBlQ(#k%kL8ER<}^NF_#WV&+CLofVy0j?oz#NV#uBxlYqSg zNURAo^cSXyZbC2LRqpqDAO46l=8wCR^!ow*mcx8BCR(+)JA`hf?(f>W-3K7)QYyjmxnTEXu%g_zg}C0V9S zZU>|gNI)Gb1DXV&;;U;D2`CyoO44Iza`YWeV!!YR_U?~T@2gAFu4aw?RH+YN(cQoM^MGd6wrg$WC%DinqB zH}`-^q%7O>pO1V@m)a*w`RFErohvEvw{p#FJD|($7<;vvo%Vdy!KUu;PrFhPK}>Hc zUc8(%1Aj7bsC0fYF|Qd zjL#iTOKd+mVhi;p&4% zG7S=&Pkyv=#G)=ksT7Pv0k}H@I$v7CKZ0sZ=o2THHmrcnN!?Y_sIT zPn2y+?aVmISC*@bA5s8U3e0JRdaVa)w(UI7fPwzVihJ8gbNy)^Zd+-K!BH*72{1!x z;y}NxilHmqv0Hr&O$@JEP;x$DsaBq`XwhYp?zc0Lrjcy_lPx9|1rYF>3zJYrlux{+ z3>7p~?qw#x-~6=chKleVAh@3qrVHC$Z`lzIl8LAyza*v%gjXqzpW{}}Ix8beY=E$n z8Tjq?(OO5Wu`1(SC=J9>&Ge2Da_**<}|U{0NTl*ok0ITCE-_Cr9d=C=@VF-#&_MDwMSbeupEzwONQ`%q4H++k9^ zz1om%#d6c?{*>YXXW|A<*d7kL8`s${(2-TXkW$E+` zHNx72GOe++^Te9&$kPn&Z((jEvEx>zd2EvtGwzVBczLqDs038z4|DByDf1hOP<2hX zLlDl?ORo>`dq?uqnm@1%Xd%0G=|!b2=@%>lfRgZU!R$-)qxRzAz@j2~KVC}b%xCNN zFWCvXuU=)dcbW5f*}_^48k&r!xt4T>8P01XE5J>Ql)-@PmnAne>jnaOubUoBYs{CZ zs2i#fVn>m??`OWJ+dOhAa5&}D_?EkgAeZN%TSaz(WcV-)ViT-&Gr(Pr?XG z_x-q@cb4u-J-sFYx|2@e)XH&q_OSTpjO?N6grcoIh>qgM;uJMQltCk_L8TLn802hl zIhwXsoU5;#uan|n5Tiok)HQ{6L)<)*+6u;mK(^=t*{y2X#CvMCsiOeLPgm)`fd!5pRpG2M9ccg+6^C0`#EAMeqV;8K9I$sVAj zr<6D>(Pvn7QVi2sEAF8jqEh|as?(A(=*a<2)j2}piBf3y3BvkGeObyg5X!Gm#(j^X(#TqYWUW`L&8#}$$&m)4@$fD^5hXWJ zIna3$bE$hNvKCock|41dqjjPisTWk&K{Zun$HSnx%=!yi${?5WV4{3llh ziuf`Y-gUL%rtLW1SSY|*&Kcd`Yu}yzV&cIi(>SWyl3Xmhs4qnJ0eBi=8GEt66+BFd zio7toeKti8!fqI46!mx*X;Rquowc}6?nNgca+71j_9%yDbeL4|o8QiCh>2Ui53=y~ zJRPzDM&hefm(6E&a7xZ^M~jTO35PHROs!r+Rxh{nb#^uLZAB{J>xIJ;SICv*0CnjK z_^kgu`n}D`+R41SA0|>ka77H%N`dB==ua^+5d?)^&|W|A$ux#FI(-_w7}1%2oFEOy zqm8H+qjZEK&oB*w?NY#C*+d6>QzrCo?BmWPutlJs6|%SY;d^VFZq(y)Dz!`Qm=oKV zPe+;gqc5hpkAY1$q=@GdlaorD65OpkEaNJ?&0nUiz8ezNo;DyF7{}3a8t;GPzCefM zr%XuGc$Qj$K_=-j7=?y&QL!S;Rf?2I!>$Az5@Z}nv-U__y0>I7OL#1i;xZqnGaT=n zm^h4-Futz9vo|7vu9yV2S;6~o0uNAjEUOd~t8ZWYJCSv;OKsy&KgUedHHVN>lmE$6KPL_Z(EJB6#5TeLa*j%&{9mctLPi za3*$c2!WzaPMf?yg-kE_9+lGDlU9^h?ls?c$$owHQEo1}UOo#9s#qkOvBssk)qmdq zE#amQ^?l(CqAa}?)q3h;mB@6qI3pGMGr%y%up#BP8}G9_>M0%hG_>bwE#0A5x_+!y zl-JRfa`W1ikSA}Ev*rhei|60lg2pSgCd)^}?51Z&cG6HaW1*!Xh6UZ?GN7PH{vmuhaeXCV&Uu-_cI81plZQz_4c!rQHm{AD{g<<@vuNBb#LT_G)6C!J{9v`{1(X zZY6|bNc<0{5f$~ePu3m8*>|=e*WLJ53ds5MwhsBbmTuDRbdjHV0qC+sl(xp_afH+& zqw91lQ^}3rWjq-%rPRCAm82e!*@RX%8>pUrX&IRlKHj?gB0-vuKDePF)XFCkcb*oy z=LZ#~-mTVM88fkb3LkRYCOIxLBA1gFV&g1t+z;c*UGl!`(!7b1vl!h@$jjC2!U&IV z#@*?(Xz)KY-nhgC@rk}o#^g!BG@BDP_}Yh3+Dv_(w)8Zy6Rzlvg}Bvo>Ir=_d)ToZ z84+?>%9M6}{i@k((FPtfIgK5b3_s6OwYGbaOx^1($Y!f&o9HK>If3xCt>m3uX(c?D zpxDNi6@*Vzbojx>H7h^sXbvq^4Zenh0$u=i{jiEzSg>9i0U7#Aqa4`N9yrhuTN7#A zg^S7F60eeVCcE?t?X*mE_(1#SZT)7aAo`=*cNIEP#(P{FW~yqbn%tZDByLoC;PBPZ zF}t7ez6`apvu{v|fSm^VqE5Z-_49TpCaxa$r+IP_nO2PwE1HhnG>Y)>^xym)Xdbe1 z6o0HWAU~iCuDswkA>ic%3p$r(qcW|_@&mH(i3q*4UL-BkDGxml1+m&p==k?xiMkdneMmZ`iFd0$ zA2msoiGe(0yb8l1wGrNg3PkU<1;_RZK{Km(_o*mN0WZgM4pQ7YQ9T9DoCr;kQg~2W z!4jYhm3?#S=dAbhwPZ9;w1~IE-E<^A(e1MTHw&PTETsm(%?7b?JR+8P4vuT@64v>QpZvt7e(G&_=P#c& z5_B3>OE^um2;z7N(mRbiesfrlyfFKD$;usV_Vtl+4jVm7<@C>kx0Q2ME$4j&D|gh& zE{{HUbVU^=!LP>b^rGEcd(%qWX^HPnF3JC3-ZtqbEL_@;fZmXHcM^u~`+=$gUZ1@b zB75IVt7pQN*)ezI2X#O`fEBTqj{;ebR5QM8jbGXIgVkO-4Oh}})@&XUHQfEv_{j+wA$+-Ma+vDH1>nmgMNLJ$qyF$EVMbZ1pIuZh>*(5So znT5LJPRh^7uwI9oe^#Kq%k{x{zI6&HZBkVCK+>k4qm&v4_yP1gz2`Kgg955Q)cIzK z)pRCq+wkKbz;9kZm!n{Y(>uNMn>B)XU1iqJbtW@36zXUIANNh$Gi`}4eD+n6hT z69l4_4$G7!ME9E%2vgv%B-&y*!-vGFcY>K&L{C$s0e@?D%2{_avpb8&Pn(ohb`A76 z030S<11%tlEqzs}-A+*6Sgor@b@6hrL?alsjJnZ*|zDkPQkf zRf^3z;M#=8?5&=XZDvT)9rrDiEm~=|DbRvhJ^~96o&S_t>?#oJ9$*(>83A*@KJDFH z3H6m<*C)mdTXv|6N#|i?oi(RF3TupY7;)>4^R^%|1|dMI(rhP2y1B?ak@~{^ZtCXH z(iO#P>2rkNj?BF;d^vH78!Q^~wLp@21D zx4v33X)`(GTtxRi5uv)OPJfBYGuP8U)bYfJ3eVGzq5H2+;D(nhhvqT!Jc0z3?F=J- znf%=$wXa9dpmNl2(A1mkt_)n^*$L?HrEfk1?Wz#e94bnX?~a03s}f^ubC+4lI{P;ZqEF0stgxxMHEF|J-} ztc@C+<3zb(@82(>5NK^m=IohMGUubK7lOESBRm%Y_eiAMef&5Z3$lMni;UEM#&~UR zm%Q@|CC88s3k2V~lKA*sX#+EJdQSiOF_*eGbJG2@2lcb%hu;q#4q=l$lytP`@emBHoYU`yYfAKZK>grj?zXiiP zXui?;(W}VM;bFLmYT;JPskp{WfXrcyd~(y~#S-Ngyaj^jq(2Qb`Vm+n*2p1>%{;04 znaTZ4ZmMh4}6H?K#(wjV7^@wb%p7Z#wb4i;Ke#b!wYqA~?H5X_i{I78hRgU(au zHghxyH4rwuW(vBN-vg}ZfIRFHv^Ew8EbOyB#eo2WNBgyC-ie{F zgdhr(N33DS9YH`usvok?F~Z8Z(kUUg*<6HMXpCVkk5WxLiiQ-onyhLAusgRqLlQ9r z#85a)x|k?3ACV?;!Dw< zsl*8OIGwK5-QUSJ#mFIpTFj|DvnlSwHN)Y#py%iNwnBC`Hsq|c);Snx1gjq#I$bkc z`?{p=R_}c>q&uIsoHvSxmmirD@48HG%jQ)OHOyI#3bhLebihTId`b*Wqdn^78TO#} zRoq>KbVKw;0Xd+WFqtfJ0eDplSSg~P;N!afs1!>Y?2)DonKS}?D$?x1?VDHYxnErp z#@{s;2yRjSc7&h1+cZ?+K(dU-K;BS?3zwoimOJUESk0 z@<&m)xqB&T1X-fLGfg(5Y0Q*WRh`pM*#7vc|I?aVl|qx%#i3p_Q>Cths-1oHes33w z4IJvCS=3X#c75MP68%Jk$sRk^3Z8*MbQZtEM%{@lfLiL!S8~M z*ue}qaRyVD;cSQzv`l85qHWrwA>{Qb@5|O~omp>if9b(1#?-m+ME}P)T6DnGYDS$( znH>HW@^tP--|IC;gTy}C%=ceHed@g}C`pU!`T5rDv`ZEY#j>KS)~5!duh5vR$?=^v zOQ-jK-dErN|4VQ&b@BN@UlM&tF)Gf|!x!WIylP^s z3&mz;Nq|v#Uip3xa!}kRe9{i7TeciiIEQO{2fvnk3x2bkS8t;o`~m|?=RYHLzg|#M z^<_c1%!JR&OM~gIEj-7T{ZgsUNW`i!uV=WGbH7Ad5GyrMR&KGbLYg7Q#_51(D(O^( z@!%zCJ};LN%n8)XESK?JD}MIFP1fBIgSZftrABcvtvxCEY9{Xqa%}->o2F)j2Of;_ zPuTh6THiu%-)h~f?V&DD8ire{DLyVHt@p^|XY4=*&@l;|0 zrf5^NOcZD^scG*u&dw%t90#^s=br$OClRX|4;cRFxvA@2Yd!_8>^(EDkiV406HR7k z4XVcM$Ddi5)lK-B3gA?QHa>z$Pf5)yACLwb>lvM661(#W%+i(4Cyj66zwAV)ohtpG%t1JR2ue==ks9JXA;mh3~T9vvoc>e+DQ6DoJ$ zpEfR*hmfK~gT((Gd~!mnm2-+)S;gH*V%$^mr{XQ4vGiChpO>l9lSlVX76SBUtayG8 zuL4=sUkx+j)f-LGy4QcrsZVGSsDhBya0kBmyb>i)Cv<`OFXm#|k zOl(_!Num$Az!BO#XWwmX|2&q$v_otGQEtp2`jKjpLu{I}=c2q0CDzAo*^ff1!COLie0x4)+7yWXHl(m-vhn!UJ1S`3h1->GV$eXfgUr~J)S=8 z@A4Z-F3<4EIy0sH5A-lI*0ClJmyBCg7%n(ns$NIFM4)93zJk~9tV)||YEk=>+rxJy zEJL5F^rcP60oTVsExOQ(`$?fXa6$dXnYu_Up=c5B=;pss^}c(^>d>B7*Mt97s^7}M zsbN0(z5G}1n&?SKWP8*zCvok|L_U{Ax}2^25=kD%Ws2hS2w9`S#j5y<>7*!Gy9wv% zh_is5_a?rNXC4`$V{YF8>&*F-&Th`HY7e^!1%e*fYgC)|L1R z^gLG6%JjGT;-nwm(l%f;w(@IG`_*bXD#UA~N@o_Ram-*{*D)^eSIds;Tlm$#EgD-uUHg4g z;!$eC9Z!sTeDSPO;hoN3Iz*^ahAUhMX?>vKI6G&>4o>iG=B)6Mw^Eaa7ye4{eB)N) z(tTzFd-e!xeGc!ug=U)0PPez3WnQ}0(RkIr?3Dj(DW{>m6}=#wH9j#ja1oFr#>mT% z!9V6@T6yV!eWS^Lr6PXtY9WQbmo2&51YU`r{$&voxRI6T+|9X`Rvl@P&kMdrcmCrG z!sRqgtu4hRay64n&vbs~`I_JO4^UYOx0uwmQqMmPsJ{_2q3SIr>PGXLGLpPj@@+q~ zEuHq)5Zhxw`gN(*GYiiwEjG&MaaoYmgq5+3=W_A0a=$DR%>@k#T7yPx%#ENPW3=Uj z@i2~`Pk@m#oNen52vAtNRDVN2V1=#_^rnHO2_gX>fE7g&GV43GiroY%?$iJ|OQ7P}F-qr+QQ+pq$lw zM%7$*gw)_J{U65lLb^S!MLKWe%Lc(>he9(E^<-0k!;sVM|yy#x2LFiT(ykh4jUTBUCWI!@-P)Bs=;KDP1oz4c72 zw;#R)uJz7}=8@xWO>E0Xu3>zoi_%QDn7m-^<)aa}`l2 zs^F$j_{MwUn#rBX#>!l}E)yzzIMCTP^Zli2rId27;Ir%RnGOPgur56g9!-&H4-nk* zd>-U(PhlXE^nI<^a%^rmD=W^1qR&G!urv?kNPVdNH!=SyxfRzzwfC%QNe#Og|7j z+G?f!51r;~!sca5S_#NA66s|&vr050fE2HpYKRuyzI%*ht>QYPI@wH6l(2Pxm#kMG z+_se3cPi=0;qyN;8*^#LXWJr}(bvviUASotv-Hw^B6gPC>opum!6C)0w4ny&d5DGt zKerVqVtysWjyluu@YKa1@qfr<7XyXsJZg?kgA3tzTFNU6NfCqjNl4j=3NI^E3QU2b z;sZb@bDqm1<87xg*p-Mce#&5W+D)4pOO7k5oPXyH|HD@@B%k+<_ZvAn5S15QQa~Qu z%hlV~-SGK4Je#J=)qjEnI-_2S6cfV}^S(Bj4hf}Jo=B{JtU#U5`PW0Vrxr>3P2kE$ zMW>6LCM#VM;ajs>6<*uWf$KI-!!UEytRR;u*UZIe@2)(t+|6Uu>_V_X)83sk{Ytjy zfsq^QZ~iQB{Etq{;Foj+Xz9#dRvKT$#*m-!y(oqKqOF-FitMV!H8~KJr zC%bXD;H-~X6Y#@%5j84-u9k|Ax69?uaevLkf5+-YQ*d%rgi@n&jmTZi&S0K^TlPtd z30thGF7(mdORYYZuAXKiz8BWLxQj-sYI>fP9e7Fv{LnFIJ`vEIY!GLYxH#FV{zsBl zD}TUt`%q?zoACBXW$`w)@Ti}4_jp5Ap8q&`R%@mf1?t#=hR(>KD=7zU4n}Hh@-WXaeI(}BxMACKUxFuR@n%j#^ zwN&p9lss>ne?%ky>&}d+16g$yE6)lPm;)^u%Ln6&3l$V_29U)0_qBp*Pf`qk9Zmyy z3pxfj{EATHjM3l~r7oS2jBss*#5JasD&DqQbe-&QS8(VtgVgVHd0t+LkL z0x~<8@+Mx`+tH_*h;MjO-Kav$;A*zRNBRiK77;aUrXr(J+AX=zsJ}{ zHomG~;+lOqcq2*!Qo+~LvsZ7ts>o;S?5rX2sy{T<{D~D#iIQLepLWe`1&cvPPbY1T zaA&V8Hj#Gg?Ra=FuUJF|tT!7>;l(Xq_SGZ!99e!@V?}f2>cm!kdqLI12~7n(sv>}1 zZKM9wjsIgql~gF@S@RiBZRRA=El%D&D{$F?D!sOv-Kr)X%O8-e(cH9KcKNhPuZfb* zBid&&iCjqr%0gN&V3ydUOz1CDx+QqDnsX{}-{F1Rr9$n1lf4^3IYak5?s)k3kN6js zwC4?fRCdq~tV32dso*Q82|ewh2%{)WZG~+ZtIk}k{vrEoB>>}@-UL?_Nd-vUD+f?E zw@8qJ_St>WwLchh%(lBIR+TgX=ItO>c5$7iDN@emXUgGr^;MBk`$P5fPaetCBgDT_qcwvV;vS)1 zJX#~;npBZ2lD_ZbrV{IJr=Mh{l!xZJoQ+UlU8NQyM%Tw4w%$=DV!a)HPv7^1*FWe6 zs6_c1U|=NFPxZ)*pRscLK;w3>VKe%rIoqu(SuIO>rrx~FuSX4G75W88r#I2v%~w&X zMas3G{7-)G0goa~1iXcSJyR2<2b~%xZfouSSEBkqu%n#YgJ{B!&3?np@vrL+OIC|h z9+ZB}-?ue)J7mxhg29Td68$!=a zXz%IU%Kyji{P#Y9|K6tXua`Ys8BY%MzbjCu6(qy%)Oena+4}Q?W!!uc)}iJC9#EWH zTfo%V*{wreD0M?0vfchoi@lx1$g#JwPHss_C{yIxvTBbU_&9R^SAGOCGd;E4<8flJ zM@Df0muv^4M1rG#wHt<-k&~4QJ?&2|j*QF76r~N;dE2#bp3+%78k-+7T^Isn?V#Lu-jpszk5J?RHZN74@vP<;Uph z*X{#|b$3WvxokB-C453USqj~6y!Lv$Ej8tAoR`K9ooCihxbFXJ?@FVZy4tk^h!usl zR-quIwkmaCk{N_lsnAw|q5{ek6cL$0gfN9uL`#(+S}Kt-(kO#W0!n}Y5fLH?K}j$& zgfJQikN_cMI2ms2tzQXp*ZpIo0r0MUr~!Dhs5} z=_;JEV;z)Va=1o#g%n-Ape3;qnAF?i>KXJ@q0B zDxm)9u;8|#^Xs{eQg~8m@T~e16haHT19k4#)lS=IJ0*tt`Z|g2YD+_}=GIi^`&-Yj z<3=MgOKfrX_tNnNccSiLsZ}in2qznqPA=u@wD`h@mOwS3dQ)EAJ>xSc)inuSm2`*O z`l%&G*?$smbUOW6=7s`q`TE?keAKvfZBvrfd6R!!E|o_BENHhBOP>R$GbMXkuY0rwJ$#5g<|gdq+}RlkIg!P0$eP53I!v-pF5=zWU&@-5B`U8}$6>uWG@5)|edt??8VAo+ zfdE6@w-Q#{o?$+i3nUSTQi${R<_4q+2-H?W1w8!x@M>n;p-w*ec1yORn;jRV!raL) zG`{kz0!GXcvEd;!e0-c}VKxLPhrr@BQv`Go5&|vcgM~Pd#Uyccxj@}|CY6Ae3`uh8 z9?Xixf~8pk62P>56|v%VMgDc7VbyJPCW)q*7A`4JD$s zB&u&xZxNn~;0s!3KT8pyaosQ|4WXnU-t5^80Cgwu@%6!Y<8sjLgTnMj?cuNnd^|gK1S*2H`O=%vOkQ(o zR}nN^+v1O^3~u_1T0;gJ4l5U%s;o9$1-fvuhh@Uhg`?lZNL+1xp+y5*B4&X)V-Z&* z^7Wk6h|0s4+KAXpCW(Tlw-yzKa1Z?|yya*C#e)lD6!lN6AnWZ`>YO$qUPWhi4i?#J zRoZElVC$CTMS#UM@(EN#)Pop(e@#iN_7L$2Ucv*ns&jm;A|c0@relkeV=>~(Ts{HG za!}<6xfBOD9Kh%FqE|oNsj`7g-@{2%&xf~P$FK{pOUgyh^8+%=I3d`@YZS>ju4puX z3@0RNnSUJO&wZ77af{!z4>K8i8PTi3?ah331mF6OREEHXL`(6=W0Hx9l zO>?&<99KZ!^fJg}~90dHFEgC3=Zf?zL6bN+*Bx-Lo`D7Y;vw zAzh%VjUpCX0b*rO(O{=eiz7c!MDt(KXL=WFez($D)wt56H$Z47h_)M6+r%FMx&E)g znW>AuAVKl*)f^hIhp&3mP=192cxeCjr4`!f`SPNs_pVgW%7-BC7Y7xh&=n)*ibuz~ zL9>G^4e{94r&p%Lf@u=9vJ!l3BN}6lj!|%{^y5b}I!3y#l@)*@rmha7qMP>SVWOi% zAm-z+EEu#Yp1*{JQjoe(WvM_E6Od?y(Sa+zb@01F_x_eKH$0$ds+ z;9b(z9!0YIR)mU_AvwpgxpH1cR6MgOyp=~1(QzQGxroJL@}PKl*p3 zk2XsS73N2VvYOmQVbZiWSu{P*`b98R@PevUGlb9hY#2TSv-SXC<#rMTAjx!zRmh|3 zmN-fpaNr_`6|K*h8n8s}1kRI*-K-`dk7IMCyLEW>_mc^KWtIOc?$~(QFG*6hf9b5x z(mU}jXCWS)?t(IzEhl@E9oanH;Q7c6ViAFcep3WxSAt$uFq*Z;pgN}z*A0!21D+Pq z@lySc1Hp+>al;umBKnH#_+6dVcUyvW-~1HN$R|AmDDmSjA9ubMVBG{yr-4c^e%^?I z8+-B)h`#ft5q^O=BCb(?Tyi5x?AM0HfvD=3_P2;K=M`a~x(b~}0g$zAtj3Kkrx~Py z>YrB*DTxn#(_I0?Vw7q`UA-j;mqq3BtpNQ#-UTBjKodA)X^=MpBS}NJtwhkN(Gy5c zuRs7{@(2mzr(7#Uj9CdZ+^!&AG@N@I%ukedf+8d!p|CsuRh(2gxAtaJVJ<{0n(cyRvJt9QVoG-fKKIDP z7X2vP_^B0-LS3Vl&@_K}jORmWJP|T{48&8)C(tlvlQ1Xh>F4w&z<~>kul?x}djS$a z;^K)JEGECj*X4R74iFfNA|!rd&J;6V%pIW8fw;qiw?zC04!*@JH$2R-qFO(r|bGL*kDqpQa%OowJv^!qOFFLpK$^eeRcnj~`QtxFA0LxUtF+UT`;> zVX#!cP&K*GS8IOEXo57;;CS+hosnzypbF!tKX{NSyiuO@R_71nB>XWm zA8L{j#%)TOtL5JR&;oAC8oZnc{r$m%+^4_l7(T2}j4o2+*5qanrhn~X95N3fh@OF! zMC!oT!h6H%8TpgNVOm>k8g0DIVgmcuC(YiRhzqD3Zv-sg=rnZnL)4|2R`y0Gy|d>G zyh^jAa;$8BV5pX1wDmyn$T#z9SgGrH+6*dJZwuWkYY9g^zb45l4dW08`|MDeo#cjK z!#!}wk?DuI3S5WasHBD7~=2 zhu^9LzNN2Iq=#K=-$e6o)INoCy7rnegbG3-AEBcxuBs4p?qDp6KhcUXWS7<%GWHq- zmX;MO?WnPS%2zsGJHzp!?W0!3TRkUTG)pVUigQS?D9A#n$7N8YFd%7(7Tw>uU#^~U z?jG?T8yT$tQ*@q1(`BjP$>{E5nX5_x5 ztPIPGMvULF{9e1cwl<4FYmCky^W+q@?@hdWO8+0;MZa5wqrn3+o!QBu{riMUcd7yh zxAU;pb8kVqwPr&unw zG7}nT7IjW7?HIva9cT;PQ0|8u%(Q&_rFxXT(E6f2BSU3_O1Q&-#xUIN)QnBzY>n8Q zw{fE-mlNv#&C#O=#f2~I$ULe?^EYAGp)C&0T+m3hwQ2m(Z^+ckeYC^1O((-MMq0BQSF9!j)K4GOJE9rP3HhF#@TSh* zaty&WirRIq?+>uv{=8*bATWq{W&R?$fqW-^{>sb6pg^ZG8@uMBGHJFf>s)1S+!-p< z)+$gy40N(GWJI8be;DFr)N@V@8zO@=kB7RQ_Oj*-V$FPOYNRH9mB1J86#DZ3Sp4yvO3rkL4@dSjdhpDNU}i% z&hj852>bJ4RMSPz4dDr;GkcZ?j|$hyvp%uWQ1c}G>@*%z=|Wxzo1P6 zF+b$%FkdX<(voqvLgvb```GI-OoVzET#v!{w5-xIFK&~f^&~oyya(`}Jj7!36oJ=H znC>zeva~LXwe%}$gDa=T==DC|r?f*;mE$@Ss>gVFHU1(>DRsI`E31w(GjkLUO--=)t5nH-pB zVB9vOen9tjVEyY^dRJJUYyXDy-ayTko^_T7mS(zMAs_jxz}XUWg?-KxoFOC3EYOG% z>Z)@y>*b7Ppkkwm0qu}x@aAU6@TC}7^Ah3GH&5bc7Ql0hNi$m7lyuG1e)+pT22`^{ zGr_RrT%^tHv&YGvE)$xl=JUOWbR(wQxVy0$^GRL(~#nrY#(CSzLV)-Ivhc4!m2(t zP?qGJcolI*m6OZe1_fFj9K-6GsX6>p9#WZP=|?8^28FP@it08U--3P|asPN@MXrt_ zaZ&h9+n4IzXX~4(Vq6-;lOeQ?v{~I@OMT?{(YHnuz z$y%aDf}xlul`q6QFD>6uJ)0v_G~GFpXUc7n z7L=Xu_uUYYq~XwUFWT>}b!tB%OUG=7(!AE`%&W|~v-tDo9~@?YWMS zplnECQhylY`)xAGHHjBXH@MwlIOyibFdiN%u{!P%ww}r(v?|2xCY+Sjw`!bf=$^L_N+`c>`;-kOo-^Jg>nxHpzT$cBGjgp1+XN@F=`(d=# zR5$%68qUA$Ny`1<9zFlu6&coIbl3f}Wb+kXB3%zPo3gF4S&!VF`(30A(Lfo?^l}q3 z-{f1>oXX|@N?-7T!#x&k)`WaX!<%t-Z{F|siLZ9rdU#IuZ zqW;Bd8TkR_epgAvYAawp=kKxh4rfL6N-Y1&lfN;p%6e1~kNjh-^&x8Sp$*j!ZuA0^W_v^9%KbeUZOns z6D&n}^ZQ~geGmTd1|H2}aIt_pc!9tjygH#IZY+Wy9e+L2a6%^@zs&~m`TI(|)2z#V zywn2LRaqFvlA%Nw>2dgq4BFz3#U&>tjQtkDGA#V@uwsfs^g4|FMwzSgKd^qL^yG{( zKJtJ3lS}r|#r8koV%w!&ZN6rW9QViX51;y9;-O)`-0u9_jo*gIG8_VTsYOzm&?b## zUs=uuk>HO}(%FKsSCqjzjY+MqI9R>zos?J(a{GrG=ap1N`W;@5S}n!uN2%y^*)-~U z&hX?s8~EQRDayNUlo?OYzYT!;_>Y053#)v)FICNsw%1x_&|=4K)#DilWFEh$#z`Sm z_gq~u|2+MXAFs$TSiB4-8x(wMw?%di%hcsZuAqN71U@n?y(5Ckl_88uID*-*M*10+ z|HW~=w0&}Y?Ozvu*}C3l%eybMHl(P%Ywb6y8og_CRQ3GdwWj)BF4?yDuL-efj*^%Rc7bmwcqnnuB@=^|kekwe?I+>6@AB8Jp{y s?$^^Z*V9Y?dDmr`8-ha4`(23suWuOLlJi`8!}} +{{}} {{}} For simplicity, necessary Kubernetes resources like Deployments and Services aren't shown, which Admin and the users also need to create.{{}} @@ -48,7 +48,7 @@ The NGINX Ingress Controller pod consists of a single container, which includes The following is an architectural diagram depicting how those processes interact together and with some external entities: -{{}} +{{}} This table describes each connection, starting with its type: @@ -99,7 +99,7 @@ This section covers the architecture of the NGINX Ingress Controller process, in The following diagram depicts how NGINX Ingress Controller processes a new Ingress resource. The the NGINX master and worker processes are represented as a single rectangle, _NGINX_ for simplicity. VirtualServer and VirtualServerRoute resources are indicated similarly. -{{}} +{{}} Processing a new Ingress resource involves the following steps: each step corresponds to the arrow on the diagram with the same number: @@ -164,7 +164,7 @@ In an earlier section, [Processing a New Ingress Resource](#processing-a-new-ing We also mentioned that once the cache is updated, it notifies the control loop about the changed resources. The cache is actually a collection of *informers*. The following diagram shows how changes to resources are processed by NGINX Ingress Controller. -{{}} +{{}} - For every resource type that NGINX Ingress Controller monitors, it creates an [_Informer_](https://pkg.go.dev/k8s.io/client-go@v0.21.0/tools/cache#SharedInformer). The _Informer_ includes a _Store_ that holds the resources of that type. To keep the _Store_ in sync with the latest versions of the resources in the cluster, the _Informer_ calls the Watch and List _Kubernetes APIs_ for that resource type (see the arrow _1. Watch and List_ on the diagram). - When a change happens in the cluster (for example, a new resource is created), the _Informer_ updates its _Store_ and invokes [_Handlers_](https://pkg.go.dev/k8s.io/client-go@v0.21.0/tools/cache#ResourceEventHandler) (See the arrow _2. Invoke_) for that _Informer_. @@ -190,7 +190,7 @@ This section discusses the main components of NGINX Ingress Controller, which co The following diagram shows how the three components interact: -{{}} +{{}} #### The Controller Sync Method @@ -198,7 +198,7 @@ The Controller [sync](https://github.com/nginxinc/kubernetes-ingress/blob/v1.11. To explain how the sync methods work, we will examine the most important one: the _syncIngress_ method, and describe how it processes a new Ingress resource. -{{}} +{{}} 1. The _Workqueue_ calls the _sync_ method and passes a workqueue element to it that includes the changed resource _kind_ and _key_ (The key is the resource namespace/name such as “default/cafe-ingress”). 1. Using the _kind_, the _sync_ method calls the appropriate sync method and passes the resource key. For Ingress resources, the method is _syncIngress_. diff --git a/docs/content/overview/ic-high-level.png b/docs/content/overview/ic-high-level.png deleted file mode 100644 index 817b2655d18c28e2577cbb14aa733b3348e5a201..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 99912 zcmeFZbzD_l*DkzO6vP_^3q>gf5fG4;#sJ)OmxPLdba$#KN($1kLAsHW1_cR8X=#w| z?mAGSz8A`1QJzPxSEY`>kY{Mg>cQuB+7^R=Rnp5Jw)tnzTZ@-->? z(s$Q0uU|*UR}my%tHRG?P)cVKclEFos`}c2nXlK1c5p@T!J=4V#3;Bq8 z`1IDwE^`CrLl(-9A6Z%L$}>FcLs@mVFrPl*MfFYWtM(b^KY{5WKd$%{4UE`dQK;jW zj?o?czIopjkrwjfOc;F}`a4+Y;I^xEI3$NJ?P>mE*n zy#eh@{3b+Jp{kF*wl*`(e-7A8?Z}l$>!v>8TBozU2P=8!@_!UvcSoB>dxb zx_+MTRS!C4rW5?nrh?-a<+l!Jf0lJIKA9iDj}oa8p{3)OWne5;CZ<-86P6$eRcCqn zrJ&z9zgq}-`?oFbVcv|uUjCLYyPPJw-r9F&MPnX*(;rPs)1E2jNS*xHJND6`xqNJ3 zt+UXiFFN#2uy@c{O!$keyBhf878px?gC>r#jO{@#linos!@LEnw~Sle)c9vcj_HSZ zqAtNJ1f|Ub3v=%XjFh}?W3_wJo_gMdZ6{Y+s<2Yc+$sUBplM6ET`=W8>#8q(+*&3ZPc7EO%u($goPr_ z$p=%j*Uq77&UH*XGZ3!@#`i~E_4eSlO4|%6+uJOsKplM*(FnYDoZv;ig@jxL#1<#mL?uf?((}}U0S*=1tM$UH*!nj zJ}p&>j8IK8YR%J#Tq(knDun^0J4*}$0|P^ydXhwrq{FKeDt6EmDxX->r<5&W(J95T z(@yld&p5Hpd0VcZY*o3euh4}VxmH(KmuoE}8A+BC_vO({+3xpQIT5}Ghu(o=NP9t5 zd4?c1MGJH>o4i+ded^|qf-~bC_LWKcq#Hx$3w;;ZcUFm>TO0mXLGG&zDpsoT@C|j^ zPNF%QN$!QFk!{Bn0db!F9j@k3r_+QF`WW5&o@jE+ZE6enu6c(PmULU;k64M=gjPiNccz*&} zIn2EZ*R!$gpCsJB@9ikQ9IpPFgLt=V_B`Q@zLP7BjQ8g&v~_}A39~Fl7<0s=@AJ2% z4OQF}M8W~#7oVWZdYUM&eYHG`&)w7%I0aHY@mgCt!IU=N)pEoH@ow`erubK!&#C|A zL%`F(cx(-Y-78nC#ouH(VjrpGNR%mYxU9p}U%8(qNF%iOxlGDljDu#0e|2e^{J1>4 z1a&O6HfwP9{6Fhjyth2U%z`h6s8>v>`YgbGI9wthD&Z?ceCo#^T>m@x52AaG_t(i- z@>TF(52D?-e0dy5b1Vd6b*_t?pb_NDjgsjo_;R%7w7&jpjf~$H*NK^6jMsg`rPmtZ z<49Lub5dyk)+^>J4=RT3r+%=!A4LD@^^YsYeo4>?ern2-4l<&c$bA&5{(9Ig=M~&t z`<5Lq;_**~ah75nw}Nkly@&}8{o`F3148wemzR$j6>~PWzKUlqDHhR9{n+2b!-uZu zX-P#nj>|XET;@;s!{EoXfI%io3N5dX*w0zxlgx$+fFK(yG{s4C#$pBB8_QPDB zQFvh_X5-RzdA%ugJk?ju9dn9WA)4H-uSU4y>CZI3KMM@vz7+Xw&npaFa!023s}XD7 zJExN;=@=Lk{%C*D(LC#R?`V(dKX=EQ7pkeQ^`A@V_@c9REB12qt*{>&bit|OsbxD+ zdhBT`r5#BplRoG)hDUC1?M5D6=}9_S@bT|Pni>>1_uJ%J&(Zxd6VZz#cWHZfF!h_+ zUHF`%BDr->FOr)GHdtJG`B+_hoC0L_s&#R|pFJV5>nP2zt!DvB&{moda z5SxvO#eJA(@|qai{<)T&vI>3~&?XUu#VrKVr5YYV=AiE)fn z&7FyDl}&{E`nb7tWLjj>mG@)&jFT0n`8*^>orULKb|pvl*cfHDw#x2plpmyj(z<-< zl3vyFn0ax(=**{txUXoKb^D3S`Mq--#dwIeCDqNJ=CO5V`CW(9;FC={>;im$%aK-M(Ni`5)rl%ikcoJQ{~j` z{Y01?mEKU?p8j=Y{T5Fbj(L;vD*9h^X%Q+~?=k4OcjM3N>js+lUny#9C+D`WuvH8S z92Ccl#K6c&xM>F)F7Uq&F=9g~g%8FgslRJIa~=1H&9C!hkoIoe=HO8E^OMkDyBE%} zD`n7S9Mm@DPsPW&yC7QpM>t1{oHF6rCuMd+d%tz#{Rci;G`U(k$EvD16kfa-QRQm_ zEdZTRZ>_2Gf zxKqXswq%NJ$XSA{CJ!PqCM);6&cqt+e9czAvpuyFHELd%YcU}ic#(J}#*-1AI>p{& zBO~J%bTA>%Q|gePW%%pO8_O9c6ZV~48Q(=b0T85X^spt@+OPKg?S=77FYU{oLTL2q zqFs*{Rn@|gZTst&uD2&h*wOJSkLK+5tUIx-Y=|}RtGgAj1k2U$w`lH|*AF(!%F327 zmfw)c%93c-bJ@W^YtWXUPRDEeL)XsJ+R1Wcq_4T)i<)F`0x!Ezc$XnZkexF>&D1h| zhP0vKEZ3X~Y&nNfPh70f#%iCWl61A7WO7d7V^9d?_IGdRG8^R-YZ1M@2E^&Rn?R4QtmOkF6zR6&c(c7`|P~I zI&BQu4QzZ2=p1)WrO_Zyd*gmaU)zPaifXbyOLKaeiaQ{7p7i&2dYO~M7oXtvroFy< zboa|`XT_>+nG3|uk7YD>7>sDP4L@`@&~{;Ne9kuCVJ4~{{$f4(Gv;XQ+z7_Rvf6!R zCLLa@f&0@!%%`_Uz9df%?is&*V!#nhCw)PiG11!jAgi;{VY=i^d$r8zY2KG9s_IVp z6$MexZUnc*&&GFeFD{T{Y(A2--4Wj4HS;anm}fMwZ5%&!O8+$$eEza1irA)f2`p3I z%et)X;@Ek2EhYq~zje=eZNWM}L%;XU(8Xi%4P)}dJ_d zOP?&II}k?Ph%Sg=95ZQBTJP$v(+G3wyvv@T;svkEnCG-o>RmTFElEU(b>wal{kUJn z{Fe>G70lBf`(Ny!QiNDTjSl88(X##^m`6(6$B3q5{GE2e=LTA~^1ZSUjqUtI?Iwp# zFrIwxt5RGl^j9MeQXX@0eL?gB=G^t=5`(} z5Isxkm$UjJ@;YmY%n2cOPx+DBYWy>20G~yZ4JAUBjB<4a5GE0RWVacwSzsn!djm-K^vi=Q@na(g@GWeyX;gNrw@DBlY~k7frps$Z?X zt_K>j7DtHYy)P2yq&nAfiC8oC-LQ8fBkr+=IrCblmxX=BgTX#y;p&oOVYz{0t(reu zXYEzeh;)?RpvHNp+RM^+0`hfIMe%2JMpp|xJbYzEyI$lyHvYVL%IoT*IZCXa6Mmg- z?(8GTguJ4-J6>MwMdiKzZZj{&@&fosfhs(=r=)H7DejYd17o~B37ed9ovlikx`8z{ zp5V^cg|6+)&eYZ4r*p)TgCN67k&BYlZyA4{BX<8i#o_o*Q)1(VwkO@oL-#}WxAd_g(oRuT*ws*W#y zo@F?hDv~!2#;SK>*(~QYbACY0yB1OZL2g1;-#M|G3^th_=O9ry^_mbH z?zr&4w7Un#m8a9akmk- zZfesEJZ0>rnZAlJCF3l{iS8^=Kbs?l()V78YlE!m5%w)Z$GPnPeCeh}%N4_ zdnc20j~M$X=*$0tDxkJNp=?FvB$7PZ3@e>o>SWgeT~xqvpi zwI>T=<>3RQ63-E3;<35jUUq!tnkT7E#IGh&Tm^Z{_E}lNAJ+4~KLhFpY(H|=9r*y6 z1?U*w-y}{^JH0EcBOJ1Yoeg4rg-&~;Ewfb6vG?h9b!}lCU7uE)D%@RHp-M;l5S@eyO-rFgKTN@V2Z-hO(Ok%5B7!box+oEplxbbt>A?beQ zDSc3*fb zQkl6|DEq2=+@F%XBl#gqW((|QcT8xLQ+B|nO zu+p=Mv?l+Zv9PlLWNyw_wDh3_2|Ih9wc5Sju87{fzJ!@l^B}U0d4lnQW{L1T#g)++ z%Q(~Gv&O0WdyaZFJQthxu4k9+EEij-9)yKw$|T3ibJ3fNr}OYrYRqxxwL#bp$fwNg z>5X3(Rfv5roY3hrgf(gks^5kjXxZn_p9?Nr01k`zC}Vs!Ru{-^KKlOr`Sb0C784c& zWd>at{&DOEo$aa;#LuIWTsqUAB))!q+-kc0r^%HC(|fsw-7IL?a_7AU#;c2-J6G3a z3c3**1LfLp+!eyC<+Y)rqJn>p^B{`;0*xg2E;rZi99?==Ryp{%dhMRjqem|*+)n%B zMG*cjh>{_1d4$lxu2bU;p*PW@XtdwxY3UR9Vf@V+0M}~~Tjj0D@Y0!Fk&-tBowmk; z*`Izxl|r^N4dVv(+CJ>0&n%HOGc&u?xx}wyUrE6E*#E0xcMMEAJ}xdvBAf&N+_~Vk zsrG8`YYNiR(PVt~^1qw^RC!WIlHFHujv^po`?WsT8yFmHGthPx)$uH+JJ)&79xeOg zMO=~PRLuML@9}52uAd#@4<5D14A#``&eoo8PyWPMw)uQ+uv~MdD|3SQWD+ZQ8)2;J ziszY)2w%E=87{dH8vuryBDJYSW=EX0)XXwjVPWC;9WP(s zu9?hoW3QN#`gOymFaP}bP}k6qw%GmS!v&`YF;+93Q&Wji>xDGbQj@E`F1wYRW61uv zoxZ?k+;^`roKwoq4wEs4BU$jPAd0BL){Pw5gyaF+5dw7Uw^t{q(HjE}nN)hi6>l$f z?u~wuV$-Vr2aRztWm3t06f5k_5)>F55I}_{Vb$>}vYt!v^7^NAt@DY8udkAhPQv#3 zTotjon>WYlr$+_3%Yzq`6*6Tl=F6Sckl(u7jr#%Fn4)- z`=X#bDlutkceG!;xXsV6ap9Ku-FXG~%Fpg(_K)AbeM`cw9{`U=Q;P-N=HpYhv$L}w z^%J3+byyz#=^yv3@*Q&*(=^To>+7pn2p3XqoKMfg^JHmhsk*i?HUEXX@?dXc zhc{!ZiD%)lfAiyJznda-w6s63FxV#q1YC@giFevrXWwt(ciOf@>?MZGcIcP{hF1l) zd87AA6+KMs>eZ{&JGkVI((sFF_KOeIKBNZg+CuL)|9CP{q4#z+IEDK_^={w1`7oTr zh$W~oj9tv1hL?oPJT7Z54*kh%AqzH0Aw=iTIhV~7a#GPvrUExZn2#eowX0r!%5H-^)WdyWM1&kIB+ON;*HC!un+Ob9_ikJm+ zf+?X{#{E`d<2cKRU5kKadRmQl$QUM=az|FjZUk#O7fLNoI_;J@vL?gYu0;it2hyR_ zpX9GQTwf?_vK$xq%2c-bN588{pk1c-`dN{;+L1gq9OF1-pc_=l4jc1Dpi6;4K{}4) zX(~Cw{<=Rf0`BL?*X;%&_s1U;5i}Zh7ET(TM||f~kL6e~a{I~8qFq1h>r-<>+e7Jg zYPbn+m)uO?H138Cp@07TVY8f+CLraOw=V8~sX88sn+={4(OX&B^Zhku z-^~PK`JbOXl$=$QP7W;-lN2ak(ZT*sN2X>uzDtS3XPt(SXE+H1E#d=Q%O&8VWDW9` zZ29${_>%E?b$JpqWh@pp>d0AJf2ERoL;7~)nVu>m+}{+a5ks;d^KT-G|2{3U(y z(hT-x55eBv+bilKhf{{Q}FTi zhg`dNB8g$v#y=+c>PLymOK@`261`ayFO^|4?jo2bC_ zGxFeI3_4O2go&3zLarbdPfZ=)x)v#!xV|u$xi-^fP~~xHda%4aqhz^G^aK^SGSGPa zW(=Qy9HV*>5}NoagEsi39zT|zXo_qsyN+iLj2Y8^K}q9>sT(rq|IgUzzaTO^&SGRp z6w1{4toy_IAcjv=Ym+SxA%^M%@$6HA&H=U_Yd zQ7SZqf!12Tn~jX8lQ_Z~6ONrYC0{5BUL4~v&-K=wb>Q5qw^$J9ikh(Gwwg)LCs{ai z<_y$}gnqe2%kT@Te|h?3UfA*s)7THD$sfY%KHU9C(S7gd>4R zefsoi|2Un-FhIW2#}1J%R7D#86(Rt7$f&7B6xl3ffNjBs=hUCjZx}YcQchI>8V<>R zlIu^X@52w?Q;6PvRD6eC+F_wosk^&dwZN3JF^YSzV>4xJ&4z7c{zvB_VbRm+`A!2%J`Vm05_J)4a>@mD;x z|8iuAkm=PY0+lqF!_d^XZiT6{NZ*X)wvvT#U$#9>e_S5t;t`}j{_C$3QUn-Br#`Vb zTp7k|#{%Lxf9Nr$+Y)`d<5B!~jPrYqtceaMH^i3MrpM-igww$UZqUk7`!)a3P^Lh; zn?y;59O29aRj*H;@fzFNSm+93c*15fz^nCx-6HzNFH_FLlz{MQc6K(E9f^=ahaWFE zf~lz~@Xm3e7hk_C-*R*ZM+7?+ym;}LI}x_k04D^TTyk`@AgqFWrZ0gbTmn>4>A{2J zE5bLPf*$|}0oY42k{j(0+i{VI=$Un>VEn(0)Aw%9nZvn*mI`9o0Ms|GO~73U;(Ev! zTt6%8N&>#Mv#-#C-G24uylympLml!AMV$8)<_G?(0}cyFA|_~ToUy;ZQ9c9agy`X5 znG;ymJzdjl)Fi^#(Ejb8AL6RJ?5%_8`^Af|Qd6p>yVPw+ztft&I;v1k^Hs><+;H*!jLdp}Z>J3&xru$m)!rMP zxy8w;r0KF}^>%0#oZe^Vi%*XxB3LS9>mfL7UWja)3v375yW_{P6?Rm##tmK}3e!Th zx~8U#T;_J8{QiWvI4TgyG#_>uf~5~_@pD?P;Ny|~ecRi`i#ptN?L( z@#F71VP_a|K+HY)CYkr&oI)0UM1r_}yl?B#qo}HyGFBfvlhe$6S=q~fP7bO4z5nbmZIcLS6rf*E z(Q6Oa=KG(tw6uUYDNskd@EVrZ6j)v@oj>SYoKe1S53^1>k5zn%mxP!E7P6rV~7az$52#EEb!B(*mDBAACd2ldaWp;i7+vA z3heKS=8IMvPu;Gi)6jTROKPKz! zT(-ZK=>j_?7>OmoP&gMW}tQsyCtqEfv+5_mNx4B zI>wJ8JgRn}5-54oUH-dHqxg^Hmq5Xru36&!Ifzkx`a%Sx`brQRU}p)x9Nl2$vzWVo z53H+fZ~Y16#?881$TZI!O*8TaZ#p^n-uPyLg9w<;YqG=XNWb;v-+qjsm)}`hpnSU4 zT!O#o$uZSGeoXp_kWoH9!@N)Okl-rIBT{b5p&0^-Nvz%|C}ZR8+HWmi^B1Xz?&NeE zR+}$GEsfOB1GLyIfuyL~VPQhUx>#KKlY$`+7WO4epPe&*pRfg^~IFVK$a{~PQD1Tm%L?b9{zCGztgKwyp2-yRSN&?JGNNTG;@=d?Dgp_lA_$8r zS+^MjfP6wFf8$+>T%zRK>4!Wv^J$x#wvdng1aJsyyLkt)p6c&R(sJ-p#9W84QG3m& za#$KDE~=}a6M{GVHhX>*TIeE45@g1e?+D}o{LLbfC&V9_1LHYR00sA+~nYi#EM#qw`2b=WxLoQCjdIXaE40Fgit&v?x>_p|m> z^aEZr=&6PRRk7nGxW-?1F67lB+da~Irh*9P1~@wF&aGR*e;snzgJVe>hkihM!B!+v zM_0FN+IeMVh5nA``}e;9kFmo(A?crT)`L$pQk{_bw5LDeEcfjVCchdU7+B?)fM*K} zrvWIiGTzt*0M{EqT}8l#bpS*MhyRUsm*F3f`37x&L!fIPcU)?s~C zCR3x-QP|qvj;A+MIjbUZ_!9_DLqp^38tGYg$ib-4A3l5_;kHbo(gUXJ&@h1dC}+1f z0@t9Jcp0UCZCGy+0L8yi4=O3)2%I+Dg;kNoo4vSd51w@V+u5u$}Zv9?Q&?Fp$GO2vF7ez zA_`uD^8g69=ZhCsfhC+C3)bYQXalZIB9ui4vUn=qlPG~JuMSN_Z*y^&f5rbS7%M0s z!hS+JPfzMZXIIfQ-(QSSb$WVw)zwIA_`*ph5QfLyC0)|7|1 zR5X(z2m92>NQ63$_2bj=@89ErjZV(x4|e)fi-9VAhUv_N>SAP6RaFCfAp=l2bNS9M zo`n)Ew@ic=)d0*Fg<@xBmH`HEd$mmp!DQei=ztsq)oud<)kzq0mYiR$7a)!F2;@U+ z)2W&IKmDo0n6=)44s_(3Xs%7Q)hz5L3#x+<4Ixw@_yM#wk@+Gj;)G}KoOQT54ycVZ z`MpTVu=}A;f4nF1kAv4Y&&obHHLN$3g2l}9g%f0mmf8I=F@f6!NdW`jh9P+wgT=PC zwgAcR3qkNGKy6;T4&S2yS4f@kgdh&yihpLnCBlSF7y{J$Y9wZgkFs*ZJ|ul?IGO7|!uQ%yg76J@Sg(JD;~m7RWGPd?h??<< ziGVp#;*Sv5i$dKc#B`?rCn7~1Kk6eF^p{stOG6y&`QfRzUk$n#o%XjTkQq}P;Y4XO zv#eX-rd>} zSj6_A2k6mU(4?GtxIvH_VbO;x-NV&|1F>WXB?MRXOBHQM0B99sQZO`3>mK{;!@Dz^ z;~zJHacG*AJzm`sE5Z)$ft1Vq5wKhKt1TiOxq5Or_4umbx!WhATrz;*zFM!E`|>kz z+z`T0b+mMJt)TJj4(o-~T*f?;<|ehj(I5sWApaj)g1){H7Mrvzi%Cj0=STv9T9FLgC#5 zW&|-?Svk42iZe_}adD6Q*?@~IU9O`)wsVH&(9gTYJ#;_2Wm)XJyHs=fgY-gRK){uQ zr_{WB0^n_+akv%gH{LFrVzR-E>PWPIZ4c>iQqpa{{ml{kr7B`-GP45Uh!p1j1YglN#WS#wdoE|gVU&qB<=bj z=>KGKz6!M=Yy~+a{xDxcg?KEk?jjW)sBowjTc>B;fid^C*W^@H!=Oy@!ukV>Fc*kw z-Cf7V=H@Q;E;TJ{bRaf`awQbDwRc`#S;>3Za>J-QyGl7IBqRyRwAk6%pH57jaDCbe zu;r#8g8+kQG!~e<5(?Sc8-rk-x2I?tqXY&MndPB20cb!Jv93g{aZ=zAVH7nXA?eK2 z%!Fzlq`0WRRF~V8ineJa9q-a~fwUMU710*Qz=LJ*6S<7 z$(bqO6@i!pjRCI|QZZ(JbvPNySP*18vb02iws?K~aAQ$8v=wb_K#=8GPRYXFHXvRH z=M9tqqeP}T4Sz=`NQBdIaTQh+4TvmK(^66nNBFD&k@?u)`Cw;o2I!htou-K7WVysJ zr)@!LY{lA@;*ssNA0+UQdm2EIs+xu5A!SIPMKj-Ye{sIB&-LV=pJ?c5Xhi%2A|le@ zS(5M5kPx#O_2lCBHivU&s;-1OZw|YAczVj^8~0mIMB3=j_Z8C7(LHCBk0$whJjT>` z+QD`jOBE1n*xt`9S@s~p+`4sZ`X^0PJJjs_8Wg0E@<(B2!04Uqso-nH0})!0MrXAjpnYH@vF|zHQV=UClwF<%APdy}zAI zf_XcOeCnuK%mtb?RWZFaM#$4KXwbZ#f$iOcTCU#Pzi2SF57JfhT6ecr*!7_FkMWE9 zbW0*7`*S;>MS~J&h#THr(6s3Ph?O>)1ZzKTUNafRk~zA~8S6!_bf9JcFh9o_yhzm8 z8Ut^Cj;?Rn?y?wI&e)IYYJCXdB+Sp<@bU54z}YrN@+bkO%wR$7EmoeRMuXxce!);&ZPzw82RG`l3jeZy*uH%cGd?ka*x&M1p$PCI z;P6sF7&wVZHo#di{A5+Ngpg4kyAI^H{htpsAmAdp6Te{hUcz1!tu*nh{kR|{zr8it z5{qmbu#y$CZ6}XkKC~*=k3^R)#sB#6sN8qS0XW}NYIs{v6uO#O>YKJP-9bNEDF-wH zWBIOm|DhC*Q|$p3+8!Z4NZOe%mI8R55Aao9TAHG`KLd8FJXSGiWTuaJGsavAfQdg)WyW!Zq{CL$ur`${Gqg(aNuxWJ7s^Wf(xWoqaI>q1Tt z22o$7YXu_~{Wi(-GQ)CV6R3WQbS!v$n?5sl=lzu^)`=S^ilM&)DDYL2KlWXi7#-at zrFBRUJR{Tmk5p;BanaPRx9;wUy94W5IxPw09_nwy!vml4n#~haOMBBk{0!i2tP+|p zfJb8k90CC@zO@w$4#7-(5TL&I;F6DU+3HrC>y25KT9LZv`; zGEDs}V>IG6@ot}pY}EM#|5Ir(s$_!Y-R9y_ff$28qGEZ9my0W8#3TYy2IPmn;%WBs z&3)IWS%`;LFIEJH*!iC5X2{INq*KlVv5bo=h3jua504s3M;?P7(3aPwkqXtI zFm*CQQ@TAnf!WJ>##SHxJh%Hnl2@Z4A4AHsp|4ndL+dJxjO|vv`>KAd6*>WQN_;&vh;ou| zk&$R*B%n;%5;D^dZWZM#vijvVkuYPrM1ohVn5DF|6h+rV|`1fM|Ni?>L$t~PMy zj}pN0pRkz!|GN1exYm#P*N;87`^Y3RbVNF%@XR4ql}l)B4baPRe83fi+hvZEVEVVX z_N|bfkt-!Zf`q=o4(L|L)J6g0Kq_qpdzCJ=Ze{;m{sT9T1LcNgL5=+=yP~fX1axOi z5@Kt7=($spyYnwM9c6ha!WSN5KKd0FnW@^2p_q~>Tq}L5{+9SqD%v@JwZe0Doii6} zmR&W;Rz67XFMiP2+NPVM{ip4RF8R$z1b4B6rHBR&$hh_lz$UBCyiSxlj~a z58W!-c0){qPuiu!^SP+(%Dy@|Y!Lpm5?Kt4_{S(~_Qy@@mr3{@x44^I`hhYw*aMnRA;bdL@Ft;@PnWSLo%aU2vVgm18Q zOsC7}1iw-Js%Il-G|(m=?$Vbs%MR_UxQW4t4(!&UXNdnXYc5^S zoyVQRQ*=EbzKWqJL4+o!{yXizWb~`+O71Ox$W&PcRtpS zu@qmA`1g$*#!X%0DY zSMS#xXG#Od6FpjLWa*vB9WwnbW6^gQj9b|GxI@*?O8!eDZ=5hWc&lch6QX{K@guBP zR=H*#n!e4kpbrk}b0yZN?+IbL%=}~@-Sq6HK6!D6DF1pqri;isuHbQol+)jtN;l6j z`JdXWqD61lXQqiRhzcqXl)6|=|4P1kAxsFZDQ@`G8lCiz&F`>^jl#PT<-ATVq4HZc zgC&PPnS9yM_$EZehtwEt;=0|u#?@AXC@9s2I!)g@>aIXhJRTjNYcFXoYZ>R0Sc_`R zB3@Pg`XHaBD}dHQA{6HI?QZjU=Mzu$Z1QAYLCXMp{`^Ztd-Egi>6i@x!?imrWdarg zgT{1ia7(pyAn8o6gm&2W2QhLc8`|dapf&|F^&P*q8SM<(m12y*fyzR1x=KKR|D+1X zLGhVS8V8xtS>u_dkHhn4J6p947;BkL=dt~SQ0IP}E2bBJKYuWZrPr{({qy8&GJ{Hv z{QccB-HW5RQUE8KflkU&^Y#etmV5*Tx(#6>#MJWA=vX}PC z{`HLa!mSaPw6|Gm(v9ctcz6bsH7vP>PN`Rp-nKH>p3vU5-I{V;Ze*%s%XSHIw$ln> z<}|=Grfcpu+6h-_>tCjJT30KmNXM*|^~)y=mYUg9P1Gdr_ohb*W$st&r%m{_jd4tu zr|O2pgg=%OSZd$sef3PYe_w6y&Tzr}q`7p^<@_X*1v28*8P91g9E3;*6Cu26yTfGJ z+u=UF8X_fbqYsOlFWa{aU|ajdqD7@EIWQHSr4q2ZT&ZZpaIaK+nN5?<#44i~Ps&7c z4aYAkotqSsKqV^GKgKl>t{I@WJ(0MvHoGG^qU^F2*rB{2{b9B5j?~j>%R=_0hrw9` z*UIxq)-s!{4CeE7$%TsaYNq^B$|o--lqEk4bXE+9X+F*$>|1+1du`%xn*-{8wQ9}7 zKSlcuF%u%T&*S38S%hPZdxkp|eqPqzv*Z?E^+A=+*WF!wzX`C=^|%hlJgzJ$ip8t;bbzp zrL+qzwK3)VL#LHi3}wrgB>DD>b}0 z2VKz>Q8RVkZca&23WH_WGtOboa*hiI#wiJ`5s*a{)#o5ewC$ELHldr|;C{g6Fn6$B zWc{$|;1XNsZXU57IL~dv77x{a&L(n;Lp%!Y;vcMO)7d81ujNrstZ0_EI#F?nNs_pY+a928EXWPE0PZ#dUcw=*q86K3=Br=X z*e2)G7x=@`l(_4dreVTh4bGu1KzAkWnoW@3*N!!aj}V3L?=3vm?ROT-BI{dGCNn>R!*CI@ZDI z(DNe<)y@JcDm$NAR8P>79^q_=WW}CZ@pdljKOUL+VYK<2jhTZ2qoQ)Nt*E7xb~FdB zB)}iV&y}T|h5>Pd{n{Nj^S3gosawFAxXnma_GObVxdrNqdgV{PNxW7&Wp1!j*47(2 zl`@JC-DJGiln_?FegoYtT0n*FTAc(Wjyzite3}0s=5f(DJ(6+l^L(9{ zZ;DfWtnBF6^uU_vf5T1{`g!`Zlf&leq#88daZL*LT`A$LpY=QMPW{|>xY+qDB26*% z-&oEW8lMry-JcD3DuDK=+6yrrUMw$ED&xKgir36AmysMsr|F)&RBWYQacTmRCun>? zE|EE{wyf|Y^;xLmKub|0|Ja#2;?U^nVv%Mru3h1!3M>#-9JjIpv%I=ibeF*XJr z(FdLLqnnA+nvqu_ZL?JuVWGYiBWqZD_tC{#s8>Lz~60& znlIp(T-`8{Gc1b2P5v_Go%>4He$MH3qOX#PatkFZK2)?YC=7%Y8uC6NX!pzsrc`i|7Q>PoR%`A)5IXZ9MJI9aFx9UqS)d6PEhw{OAPG7TfvExmC_ z8`_SPV0W1l^3M)5Mf+z=wfvm*r#Q@rC|=0Z@Tx<1GthWQdjK$F+m7S^;5>&<@mhEYy$sO)ClfN98^N z#SMgY0GtVfN-FS{TLkax$u<4Cfg%~+gmkzKa$c_uq+{H49|QmgZhC-dl$k(EEoY%c z-QI9v8;UW^K?+upDcRWmSvu%ufD;vmJI=zg3e6|9H*Wli*yw_){NC_6mtPOh@x?&} zRU+i>aLM|KZ@i>E`3gJq{_%eZrh~-cpOx{EovOc{Z?8eC9%yLI&TcUlFVayqCPTYy zZna<^|1aI7z>&8Zor3f|5Gt}rMIUb!I>n*BMv=UXT0?qEpk>>8V?ZYSsDIdd4B0~{ z_d!Py(I3rAS#0$_aWvL03DM`ery7h(3Od%=Cp}Ftk3PNjQxopa`O> zojxWJaj)f`z{1)h_sT9D#UisCC5oyUFP*W0<_TbNmruV?g1&aBe`&x`l1o0}7>sk% zP5!~d11}@K0vS*QM*1v}R^G3Mp@ScH41tU^EYX1KPIK3B!J!=0bP^u>0yP77PyUCW zYe;VKb-VahaC@z=op~rYbm-9G-ylhp8>#UvAM%}(f*xF2T3UPP=2(T&FH#xyM~c>9 zxLT#K#yUJH5oi@bTI!*OBzz){S*u#Ur}ib3=8;1Mh$$KO72w}gfWr`!Bb!iX7L}3; zrWzr#$%In@Q1I8TiMspjwb^c#R?Rfj>Dj-oWz38bNR5n)1VRBD#+?L~RTc_#a4=%6 z_NMRV_1J%ZvaSW#t6SrXXK1FX~e;9=UBbGctxT8c?0} zk6m5e4mGG-ZE%o`(nhk3`W(BUZm_Y==fPwy{mGj9b8-9{^Toct!Nsdf=D`t+&1T4T z_~@l2u!3pRbqO?DsV|RgKlh}*$({@utG?W>snR)adS$(Ho^QWh%5f%$>dEJczHO-_ zTaAI4rviTz4*~yNT-Le78B2zaEfVEQD*7o~!QoV>%c5{VRz7;GlP%DBsVFL1cQSIt z5{_Th)z%{I_Z>O9Qg9~Z*Gp+O-KOhWrNGTqDZ7l;`XNWSpx6ede2~%>df96?tpSK5 zq{<5CL2@f%$H1FaFF_q~6^cXXWi}Byfd}o^7!|D?7iyb;&Znk{avo?a&KR{TDR;CD zgUT>)>dV%uEW|o_oj@pDhZEc-pcs zV5-+4rF+hTNck``ccoYd$96UL;>z-}*A_pM8xX!*CFjna>q1&t&^={N#Z-fYX3&Uo zX}tyd($M?i9Cj0Y>Edt%m;UA{1LRU zFX@7zsg~Gg0ZFanbv9#m8-;ha(2i%V{B^ZLhQ-jACxGPG7Mh3c0lGK)F7BE)UNcr4 z7nrFJ@W0)-V^P3wTl5M;K5XZ<8&1yK`$I?-$t#^gN3Lpf?O8N0{{#-pBA_TR?0iDv zJbs&jubq#>LE|&nZuMtu6Bgymj{DOlyuHpK_b9#zmG!<%-aO+$8~eS7)UBH2U#21F zD_vJE5MJMy&1dmcbZ8oRljzhu@+ok?c@OY*`}%oLeg{T1$N)R8m9H6oa-kF48+T{v zkI-+ymE!!>-}Pjz+hSN09gy9G_mC$)Z*q$Wvr@p-5d-PZp7Z;T34@PTL8vvCliq>h z<@p>YSKA4(3C4Xm`0xv_9sK{{Q9p1{$%LC7?>w2x3M#UpH5!fdn^HBuSI$J@Nuhvy zXU~36IuE_vu~2X5!lAhAepZ<1LQbMy!K3J+7GSMY6rvp zH1vEor8vQsDmj|%`+Mvj2#O>>A@aUc1mM0uLN$RVfyXN8k^R=Q)Egvr!coF(*^~Jv zDs#(gSz~gBKK>8(zQY~M_W%EuL@Fafb{U1xAbV6IWN%rKoxQh6ln^qq%O-ne%Ph0F z?M<@x-oN*ip6B!V9>?cT_&JW}cuLoOU*~mR=lgxW$LsYv-yZe_Jt7Q{rfK2R(c{}IqPV+_S1WIJxvib#+%k_6%L#7sI~fa}8Ko z^lqP4tAmuv-zyi{FOMCZG>GoC4ibC^kSO%zQ~*GP5WL>LeXEWy$8wIM(#FX7L2Gjq z>#knOpj&NqLHCnb^Ho5|=4`EYG*Y~aYnfbg+uJ}|7)OW89thwI_BGLh%sbI$N)3Ds zo=FmQPuM-$l0Xf|cGPWq_l6|w_5j?{&^J0yyV6iE!pz7kzZCk=gwUpQ*;zW`3Am%n zXGb>;<4{m4IX>i-80J-+P;1(sad9ej87gD$FNlX!5-cFyKu$g=e*aalO*Ly5TBaEd zWgn~H&K|D;Cyh$A_sjc%5nwG~f^B-zy<@gY$NUu)>xuS8nzvNLl4AFV1)DbJehhpH zifA!(?_Mja=is5b;nMI>*)8rTA=^VSZvC@C%J=Inw)q$yuW*NjW!|f;$+>v`3PR&dQ=XX%a?&Djss;a8`1pEMTl)yv-;9t3n zQh3Eb&?Mt^hahurncSPe6`|DW|(n(r!OXb+7Dc^pn@CsdYcXqWS4GC$F+zO1aLh z4YPKrzk=tZx$5;PF{-CoP|l_mK?$N>`T^}#uTT%xR-D^bKkmFE<_~BGD(-yF*PmMK z%S)`y2MEsYLJ4r98x2RC43Hv+BVoUBRK{Qna z%rA*!$7*FxV@51m$A@B^SIiYAU~-CP2{SC{=g@B^h6d7-SBwY5F1K)t^HJR{2VpEK%4Kf5jWQ#qkPuDZ zDaq)Kwx~)6Fz2^O7CYFU$c5Zplc%=;jv&b^kdq-Ko!SK<>2S(z=rbb$sG{8+Ce)_F zm+ntc8B~aqB43x0DcR#;;ir4RFgsH1?A}v(@H(}iqL+1;KYsM+bE@9qVuv$t|1$54 zGJJ?X`dZ{6l)>Nl7hbP4jVvoFn0Y-b&7*0)Svo*ua5PR80kLCeYzN=wj9FZ8TD42v z6p?3K=6jLZ*RyeDNPNgu&Fv;=IOn7v3B~i?!}fwBKc>|7$(mUn6&Aj2*vXI&L(n<7 zdaOM%!_$~p;uK{V5IKQvY7yvV>|i|nMD$bLtw!ZCtTmilKveF5{v6Q8B*XB}0ZEs68(`9b+spuDgD4pv;MySr7qjN36vg3#q@WUj z7$!q~U1#~A#fqfEd$XxU0Ji}W8mJy1D7s&>cRKhL^cNx3yHurobpG;fkr0UB<&102 z(b`Ty`TrP~T&1))pgb2x{q+Nx#m^FU zU(;Fv<;A{f<@n4xB$62J?JYPuSZ=o6-h3(;RC;V2sif&)@UwTc+)l|la%3lK+Rc_| zwqUx}Za6GXAE^@0SULFUNyMFzpPD8OF#4qy{$pW-_xo8Aq|zS3mOqn!+W+?<&rl@e zSnDRyq`-XD>XQzqq!h>Y{Sr)>Ud{ee54f=L{KmpM47$)!{>v?oSv!?ivrq|}4>YYU z1lK{dls7jRAtMtsmi5s!@$*ZA49jdP_5#I7$<4>dEJmM)4l3=OWj>nx^&Sa;fH48c z3&iw4K#XDo9B(?zIVeDIMLh*7$6SQ#wU3;{uiRDf{pw|@TsF}cveIKH4w zz}9@P41yr;5wOAQkV_QR{DHSJf-23JjEJW}02eZXyGck|3Livp)at-t1(xt8!uW&K z57`q})tq2kYb?FS$Y=u1<0wee8sA=4_?Qii+=94mr1^|?EZ)`oi=+n_;hq`zq>Y9D z5mG#aXaIU5W&3+wEwx!B$qq%=D>J7LKg7HAmM%&7XP4(sM3%1&|E|0^={Ehi@=wil z-+W%}w_6r&6TeM+_m^+Y(@zBojxu`iOVn+Z@{{kgY8|x?7aTX2q%TA%D?QNfoq!u; zG%r24usPwS93uF0XS69pQ-bBI>z~G+FabhiP0HZw4Z3`%k*q=0y?zP^F9>_Z7?dy& z%sIea(aDmeKqn>J-vrp%op9hj%V?I_$^qO`8QAs!Slu!6LkRSMt&~J4GcffG;(+!p z1JEZ6dI+j0kiW|TO6k&;!qH9`LJ9{U`cxNxv&#XPqZ)u#c;{D>yMpE;gtE7*feq0G zUs0srLIS*OEu@M9Kv0m`i-wIAs1xW!F?>K%i6fGb#TKL40O3vtNDO3hpVv$Q1#bt@ zg?Nbb#*AFn{un~-fPq>qW|V~+DukM0pkpH=A2o=GHM`{_HNqZOU!3dzz0EuRTe%Y^ z$Y?jJgB`1S9-cl8N*0#{h1M$_1ad%20=N!}^E-guhuiYcmyK*@WOT)6@D4OJG$=U- zPCIAS*41@G^##cU0qqLrQPGyzZuj0yiF0T2&{=U>Jx)rh7)#axc1V+nf}rLibH%WL zB!U>&B=%aysBp_CuwWkppyhwsq9Sl@vrpA+qWPJPx# zMMw@lip;{M>BY0)Nujk%OF~4H9y#-8J__AL= zf%1Rw;=LYOeV1DlME_X0ZTW(j@AH5@pOO9L>%J!lb7(~pM^JVnK#dNYff-0bpyOWj zx$kLr{B&W@9d?83To%AP|F>C=fF2l6=-#M%S7Ahqg8OODGqrmWC9Uj>r)FE~yMvf4v+J))>61ity(!k~82 z1c;9E)dt9=jS#*y1SG|c6341Ql5>e~FfW{dXHY1w#}KuXAMayN*_Xd3j?Jtq?%rFc z^nF7$zBlO^yQk)5=#E~aqfnzxXz}&Q91a`iTxIVy8a=v7J&G^#w}|{>E=d+K9TOu^)&_CI{!7-oXt8{` zs!Qm2S4JE$Ph^{n)(i;(LfAZlN1}(y<}5JfZk5xIRbcIHQ|K_p$8Qhw&IJd1v-jlz zS|dY8`>7A%+Z*`VQ$@4oO7oi?B|^c(kwWp_Q_BxdfUB^JW58m%MAiL%>RuJ)3*UcU z+W!&fe7LerIHt5o^!rrU;9umQ>5+ ziqynwf1q9xC}A9(L_N%V;$jjm{#ojtuc03zhk$V*payBSu?nOPkAP#_*f7%^s-6^S z9lP!mfrRu0{j8CM&)pOpEPBE2>tmXK5HOOpmrv%82!J^V-4nPbSqSJCaBw{fA)Y1( z(aOAP*P!TmeP#nNT}|F!`&pfHWof5N$wsiC!D~HPM!a%xTs##Omw@@VsvCd9+l7tFO0Ez zkA2eZaneH~ngAtR1+Ok0G=F=BeHM9|n6GHm&B{lf_UOpPkon0S=E1wP*W`wikS zt)NrW=mLR0+#8k;fV)nJ$HI@Azkg5fxec3I#$QCHc$h$bhu#6`ylMRXJddGZ)pEoC z{S}~GyPVw4fUFAR33TqtHOTBM1mQusiT)z0<$ zii!$#HNX!FU;P8iC(s2jvpUN>%+#Y3w|xIwKlSML#4Tur$!h-nRbwfC6f!!33d$zU z{6!(Q{c$3|jX9qX{)NANYj3Zjoj-*-)so-sL%w2EoKg|(p*5JZKbuRE)0fl0vehzc z9qq=8V>y0814b34OGdC6L?BWpP!qtH+WG1j|Dz(9_n5zNA)lb;?`Yq`@~p1SZ=&(g z_yd*vph=CQMY(Q{VuIFn-x=l`yy;wm10QEQ>~ELfIDPX37zxi7z!O-WA#g3dYbxCA zc-2HmaG854Wb#X<@?6aBESq@eXk#&G30Yj%Wm8go{m)*>eQi?vQ0@Z@zR`q*NA|ar zapy&Ev*tl7#Sp`1)0hrLoj;CPS)P~No_-e>U-i)E;k5w$)w~NEzx;~ldgmu_=lKH5 zIpzz7qgH?7iOWmOr#K%(M8rA_JMjfFTQtcvG~Jb*j&CyRV}0GM`m8I4ecR;MyB-5= z<V-+`4)e~;9KUW+HOq5ve*9cKmwUrMYZSNJ+URYJzPGsLQYCT600DNNB z*Nu7`L`S1CQESq19eXS`c76B_9m4-G`i`*ITC1|f?tZAj?RPE(3`U}!B@98g8 z%w81ljCoAamNZ8CXR*LbZ&-N`F`gUXR@8bC6ZWl)VyAOF#RV#c@m2ISM@_lMBt6#ntnQWka3~ zdJK`J^KKm#H|`MzWRdR+H#>EgJ})vHWy*>s-<0S~5Cl;kF6jVUtoFZbJ^)NlTo7#W z_cu1G(5g=Eo37U6t5+_IFBX?_?HMfFS54kupBu?rJ5+2KZ>gawZVw9f0b|{K*RYWS^S6@I@dH$9H=sFOG~~hJX4U_ASRtWtElpsRk2!&R7G`^ zZ#zB0fzPAu)7l|r$?d>NHvR04)~~!KMy3o5B8p`rFORoFas8!DhGl%acc*VLYRa9y z`WKAriTcgf|0zFq`P)US^dvZBp-D_1!*JJ0B$R!V-()mXvf{Q1^^d54hha-iE&jKY!3K7Ybemts&Lbyg0c`g^{)h7>0P<&CUMmV_WIX%JuKk;+<}o zp&1&>JrwGXLSWnpABbK5jiNmr#bTQCx>OCsDQ`%6gJeu*f=iRC{*8QOZP%HLD>i=W zvI!8hRAHI46Vc0T6zEe%jPY%dy^(Z^lv|G3Qo+#|>_O&FdEJKl-z}zDY^E^uv-uM? z;@SelR?8jUZtgpJI3a7{Wx)tRZ+yK!1q&DeVxL%}#5YNq&;J`?q6`DMSvxu>Kqw~@ z?KyHbYG*JWxt8vHK#*C26@GCRfiT|UsC>>*x;IQq8cavW9{lA&@_Z0jwD7{8^1Im%H0%4G*2b6N&Q*m|S}IhbRqZuanU%8WU~^5AI(C+1Xxq zPHzKvFpzk5yTjORW?vHrwz|ZGl-Ah!rkI1z#efL{>e4~5F%m!6zr*i4cj6w1@X9_ zscj&6Me&za4^P9E@SX(kFk@Pfp2o)x zK|HSKzsO#b7X=NLv2STq3t~m=bT*g>{zk_T2!H=xFK7Kw%WY)Px5%k)K>=lX<{zsZ z(9O%($R}-Q5h>y*R}L3#YlB_?UqE-76iKEAp-?E_=hafP9_z^dFw8WOzp|-N@;+Z8 zdBNdOYW(4GFnz2omojF{YJqv;r;Vr8&2;-;)4Z82x zsGGKpurRwKH(dGWhB5et!B^cYN}5+TUh>I%#C55=;Bm@J-*bUG#2?+{7Y&y@6XiyN zYX7SY1A!5bKXEXh1WD@fGV{_3cNWhrb%rLH?G-%93duKfwBq^nhY}?I-3xc_p}vnU zf-bD$e?F>shu^L+`eUex`glREy~n%c{-S&i$xOM+E?ecTRTtCrh0KJEE0%`RWB)5d z_nhPPT~Qp*3v)NmQ@s>^Q128VC}kY|tYQS#`9z|=fi@u;jx^98C|b6?uJxbb5j5Y)EyZM@>d@{GmIdLx0uqrIIhdQt6mv_u` zjq*E2d{+M5bb)xBbM%i^E=qU?m#<7NVBJGyidWQ89b4)nAdJLYj`u4Oe13@Sj^C5e z5w`4cEyC*nF#BxkYzu+@`??clPN?CWV&&gN;3#`rOHa3^?y=4fmismaQo}(w+$Z8~ zki`M)#1>RkG7=IJ+5?^7PjihF799I%jF(@8F9238@c^=pPp>`P6^(~^qntj-p6e{L zL)QxlWsKLEkN-SG)zd-53+o=?Vnbt~0(g{j3z1Oe~c>$6?Zaz({=WuZmX(YgUNvyr?40!!|wX_%u z3k%WO0RH`y0~N}SUA0aDMS3EXK;Q{&uOu{DadUZe)G-zWp@8%_2ci~t?>8X?T3`Sx z0fkgoPj3@a%NFyJc|;Ne6fTnY1zmS?kq;xRO++OCc#rJjh~5IdOytZah}0Pa!!iWn zjsbORFwGb`>Ie=CDzHfE?S^oi{rql@0IE$r-s7kgZh`1g0Rw#wL=U=v2Zpflk;?|k z>@tD8ZTPYlw3I6TDzU|F?mbGyR9xmn6gPi&#ODBH2Fb>eA}{C-8b99#@P_f;#%%uJ zqJ)%GH)I_-+J|#0QFqOvfaRJMNX19qn+iCuM$XHyN_p)Uj5LB^tiZPu?GW7mO#es8 z)TAHoP3^jJ4+;#kU>eAKL!KEftXyrRfGPtbD?oF+(}X*pl#*fsgE1I&M}*vP6K?<6 zP=q}pFO&n(1rk>EB!G*V_2u3J!e|cyrUC*u=)s#oN7iA)DK*Y<0wSHC!Jx>Mi-=@! z32#R6*raH=u09l4Z@m+eRDFE749v`IWIe(FE)f$O!3-l0hcMDWY4Eg1_v-F+ylH2W zILtJVUphfI4dHAfx`E(y0MXVdusxKB;MqWz2PViZIVOe@IN^I0DW|`|I8&u#5sop` zB-}tYR`kbQ<=qil8<@q$nL)roDVJ6M<&z_TPRIlW*;X8h8iF02yYfk(+rD_#cT7;no$LPPDEs2p}GLHA2zuxO%phyN^}`)REF?K4B?FOA>dnzE`u&st4)^nsiqejK8jpj# z&MBm8AuZjmwmk|+r1)(_3uN_Cg-X1j+Xh&s5wJW%_!I9|fb%Vr=H>fN=evZK9u_%wU1LPB<_rb*$VOg{B)RdaV> ziJO|f6knt!D=PFPI>*ta6Mg#wG=;K26Gs8iH;6ufZ3}A&nN)bjgqh{9ATF}EuV3?| zE>=Ww7|UmERy=(MDA!vV$|3O5vSH!v9rTx2#E%}r=Tg+0%0WPn#cec0i6wq&K!1!E9ZNL=p*wGCFy zGDYb6ZC&o)g(>SED1D}{ZwzXJqID@u3Mph=`1JJjI~;`>>0tYhOCq~ zj50Yp&g5xR^LLmUV5}kh0k07?&=wG%jdz;9UNpGqF80_NsE>q@{gjnHxy|Ni|u2(ug|f&L33gV;?j;360j5<(vv7bU#L5w2cptzT5U-JU*o zL8B6ir-* zCmE5KA}A=sT4ls#Th6EVRv=YJpqH302M2dNvwtYeLa#zxTXP(xli z(r6zvE18AOi|f?v+;bXI(o|SIh{qu3woK4++f+jQFLc}+hYEmEkEpI9mBq~LG6}%x zb;BSS#F5d03Kt@ff~dsnO1 zmQ5lIdO=9+=A{U61mekID}3T{Fi8_I1uoyv=ggyNy1RG(5T9XtPn_*a%~Z-%-W&7A z-+(=s8Kh4JY+{suBVmlRbRg*1nQ=73rUUj7f+#o%93QMl@!FYWX%~Rl#1rWAJ45(@ zNIoK-w^1Xr`xbcfEMTi6Adp)g#>*p>h;{>67bQOka)U6`sK=;LU|3Adf=xCEZ=nGQ z2TCLy@pdrQ1`|#crX_XE!#Cfk82r1I94>)rgysE5#OQx^YC5r|tV@xpp-U)#2SI=U z;yu9FaNp!EuQt&{&?AR1gt;%pa&6V&3-PDUt0*ek_Q>T!_yfRIxv<`J-g#8o6{5PX z|8#up=F`MP@$T-MBv9{~cRL@hFUrtk{T}sotrF_&+yIzzq^5lGiP{kkhH_?6PhA%T zWW$b=0nT&jsc)B7!S_cAc~n6RRF!b<04mmqFwL;jbZp~6E~o|r?_Am1dW}cBdhYQp z0tyP3o_WPsI}{p4Cxd||I7yb$kT(de;uJ8IOmL2kL#ZLQurWkxj_?4=+ZjB|Mhzyu z5eQo$J2HT&o|Oy+G|gauhK>TN=%lLZsZGUak49V|pls&ArWIksMZN{??44S6k%A%< zR)rX-GXIPdaDm1&#V|m@InDjBXjfFFiPNk`EHcl_Xa=ifLe@wRX^}O?>A$C^C?Q%B z*4Fvpo%HOPimu+hd-qv8mhe<oz#a>s4Iz8KB)~1VnrsKqEl1L z@;!5KIFa}IyPm8>`2ogsu@byDfhI@`^7P*%9_K9RlCX|GrTWTZx_Jg^XsBKG--yOu z?)<}nz1U7*V2^;M;fGUcw6+WW6-Va5524W`k%HTBAk&%l6JG^YYjo|Kf^NtCm(0C9 zKPP&vi^gcjFbp=*Rx2fBJ{2D`GI&EM34RYFOxu>u|N`s3AAeX)p5Jfy(=4!RWy2P7B0%R zWPkbmBL8m8JNhY$!K^Qk8ek+yd@|F_lEYC> zj3zaYAn!9syEmSylx<(J*urc&@O8oDg-uVX<^vFif_6q-<;UWa1_dK@61pL%wD{9= zlJ~SRDCqwqLJc5{pU`W~gy8Fg=EeIT29rexkLc%vmoqg$z*yK!@6U+^HS|C@x2RYJ z5n(TACYf$Rq6kSaB29g1oC!sA(tp1By*Z2pfm}?w`G>dJPCsz7{mEdJuYP%6_Q{dO zFCqS8w{^k-$+MT?tVX=0BGaWO+Qy@C4?>cl-(dvUvNBNm0WI{8uU3dXzwbOj5Lh~1 zaFQmHQ#i)EHY7Xme7Is_Wy}vk&2io2DI^h%^W%oEuJ68)446U;8?wLq~o{q zZGvF)OTtd{$+v)B68thC9CGmpK14(`#J2 zab#M^wM8MDX#c0kqM$}O*iIf-b~UGGHD2l(KbGutZQI{7)6v>Uam(sxIYXoAl2efc zyUV?N)iyR#j9RN92B=s+Dkc;deqtmc^%ZcyDq%a-YiKy>h8^y9bxC*b#MU z3?K@&@+Mlk$*mVB2dYbcKnL=;VODSa;v$~*As?2xkMYwtK4J)L2^4EE z&f9tY**24(a2Y^@Y)v9rODqysIIZPV4L)>oyduVcpg`huT?Kw5abjFWpujvK3<{mT*Dk z!^n5m63wI~u|yh-t?Cw4{9ABUS(RN^pE`#+`F#_(F3D9}uu-Sn6j`_A$?*ImI$53~ z&V7owb30sbD*jbXuh{F}qu8LBJ4L94*jR^AN}nf&_|Mm>Dt+0|r-Frdp z_^X|}HHvcJS;bE{IQ2NT?$v0S$RzguSdfR`tmQC|p-He!3JF|D#z~Jlm)(7JG5V_O zk>)Lu4w@UAX31J_gs?(4o-@|uUP$AsU3s5YVsgFU_%i47g5wViSFe^>9_WyuX!r8( zu?C(Ez`2O|qlf!cN05EROS@wy5;-e%*R3tCoy}($C>O)I%nnx+%5f<`EO$1ww;r!o zC4N_UsBkVd)_1jka8zfaTQ6@(r~`TZJZ&-}ZlrYz+lBguXTt^JU&B*AiX}dUf~U&; z9joEGx9F4#BoG(smbeb$x-P6<6GGw{jMyp+mLogzV^^*wLAs7nyiBLz zG4mF7A3yI`>-^|kJtTJ?6&aC%g&ML{I*l4z-?(bd_>fiCQ=;?YbK!3D8k2Wr7q43T zbn6`7Vd+cCe(C4O`+e2uz|ZP+n|*w-${0$hwR&*mJ2^SwOQB=EvnX1jeM*$hcF8^l z%EM77`wLU|yI`xGI!qf3)O35F2Z7**9wTb3cz>Ff<0=kw5$W&IHax{oRe|H_`_0_h zVsl9Z&(-4JDz&`Q9pq!ueYdyd(f;P+v*fRDIhTK(LWK|UlDXWKuAg*hnw-DD1T{Q-}91oN6KbJ4Z==RC+F%EFq-uap`%P*L&odUylqU{QQRQtC$gRx<2>nnm&3_ zh4TD7=>&~KLkSF&w{C50BadK5pV{bz-TIHuQCsc3jf`3X%5zU1ictNMa8#!$jTtwl zDK_bsHF}_gLP>dEqt z*m!S0XX@jHi-em_fu5k|A(u%ZSZXphb~CkuN;4}^7(L#(saUBPhhjVPdGDMi(ac3T zc3RbOr>)()bqWeY>Z+V~lCr~jzY)6LO*y+lEF>7NG__GZ8FDH*_8ME~ps~hCX_>*a z%_Hic2mD!9&#CQZ$un$Hj<;GWCimC|)7}i7!&D3*>E&mbizaX0Arm4Drto`HkaKLm z_DI@a4{@(st`5Q)iG>V-DD+ptW?o(eBC+~YDBCILRI9ynTQu*Rq#vKM%tiKlPQOj_-6}oP%n%nnPYDqbD!n@rnizzHcAXQ~%&|}h z`;LQ$t zU(cr=Q;yG-AM*?nl?3dOC8Mzj5I-NxfjD%qBX0Mb=YmF$PJNmvoJf(@PF}2E^;W~7 z9P{1w-u@!=p=0 zh87;)H5l#_A*TWpHfI}->#6WZU%p$xa5Qa8l^}ckSuLDwFH^nbF%?=(aONWI9}jt)-JBvY z$;YUBibLvt>OIEIN%!;?>rRcDen+aGefPG+7Ow?exW6}IP1%a(;tQJG`6l9#x{V>6 z{B@xaFQb9kk18?Sl0TeAs1MtJ8sFSX&tP+$(Ar~hZK642dIWrp<98J3}-za0kmGC)FlIgi96!svsL?~pr?}s==#(miYY}>sv2kVs@ z{8iWSqEIprD(poV3i^^k$zZLAmH~18WoiyzFZv zw%Wr88#o7>f2Q7jIxk-0n7b*m%uyF)2Kml-c9E&fUAy{Pbwk#W{Y9&M{{+FS)HWjW z49@`#9Hl1O_hx2xpIv8c-Z||^YFZP0r#ltAB}Ld-#Z>EMTJ&iD_k?;PZ0Zc|pOu+b z>IJM+*QMF+x;meUPbD3nJ9|ZXGXBf8cBaCL{omI=J6$84b!iE%DxpjdyP%O1jsVi9=M109-QG3gEA%aY)Ei;uHE9lgL{MvM}E?9SJ9*~>A_UzrK& z!|XaIgh}Epxg2^;5c7kIA9{+*wEeAR>j%QX?dluEbw6NwhumxwLs4rz{g}6IJ`R}f zF}#;*VM%@*flYMFxYd(!5<{nrZ!55CqjCy7ytmNNl{1`0BoU|3i$eXFy=d_(Yhd!I3FxI^1C}!wDjIKZbeZZxV}M`ym!9*cY&jyek8otx6x%PA8}7a zpv`9mJ;z7BDJQd)vx5HQTfs*0X3krd-2Q<&C>HbWowJLH!BZVQp#DuM{$UC^|!3#$IkOZ%6#=Y|EATYuoSAO8)buW)R}ma$-yS zWF07cliJn1^YJ!n?PaQqn|_3Du(mR!d>Nlnzr(jeT_%6^&pJC|?I&~cCA>m5pi#ZKuC zIvLj#nU|>|hZisvt_Ber;&2aKr=u|^B&(?Vu8bI}qgOv^Nvf6oOkP1>EY2N$m5LXhFn{IxDZT@DwN{!d z>yq#HZu_NOJ};X-dONDM?Uf-(=7on0XH-8Io98@j8hMVYS$T<*I=PQG(A$1l$T=5X z^2aRw_L%f%MIFQ4`{u}Q@h?t4`+d=IN(d`KPe`n}aGe%E>iVWWf!w(-XP8UFg6qzd z@!Lx}GxmHQ4fLKpp0;%z{pfmQzr`bW%A4WGD{R{*S)!qrd3LA#g3i8R{4qFNWF`a= zZOAJlmz5=$rQq`Pth@JBGJLgESnZf9Yx55KR?h`hu!>qSbxK^JiZ&4q2Sjug`#?OIaLVw|>E_{vDsgA~FxlI08Ez{XshkDmK z21jkC^rf2u81J-`QdrKYUI~ol51RRgK_=vU6DD<#s*rc-Yuw>hkxC*RwvN$#ZbpVC zo>+@*nJt>KY%-$ZB*~szzLV5^CO4y6zcsTyP?r9nI=%7wtY09miTw0WqG?%qb)R&i ziKn;Ka*r)BwiY@r<+QEZwR~qM&2iaMs(W|)Zlf`q80+}cqetYQ9{!%aO37_~2Fr&u zs&%S)yQAvCE_axwmsjmMl;?MqdQKO7jSnrWZaAo_DlI7!hctBW>t6hXhULsg?Fh9y zudZCHrW`$v41nt+T_48m96Ha&ds7%_2ku0^v+tG>5pnD4kI{AAa^^M3cq<=qzf#da z6c1NAzKIn-=_<3}r*)!`Cy*m^KFoR9CZskepG>DZE?0;FM*rmUWNqR>GwPcjjo0V+ z94bjP=}lpV65X`BfwBWwmQhjo++ zTL*uU@`f^n`ImXt8D`ViM&3E24r@#>aoe)?#t<=llAL>4Ya%dd$jyKK^v-kN%#z;Z z-)klP_MN?n2)~^;r!H#ykAE(@7zY!1)4)YmAC?-#{32L zW5S@+WnLfJ0Q{axOMfyP-l~-Ju2&N^!FQyp1Stgf2$7F9oO`0JkS4F~o2qi$F}vSn zC5>`)Su|tA&SPy0<9EKP%z7N#<@U$b1i+)Mu0@a1Ee1TC*paBr>t7yi-2UCNO*)xp zcr>^W?_J16QmQBf^#-Zmd#?=Yx{`i5rb~vtPb7-i9@iDKmI@<`uZ#=yjX%cZllv~6 zTg!6HbloJdkhhh%gye2KeR^it+tx^Yr6cQaH^Q=l3c71D-n^<+{BeNNNBwc4zSZam z5fL^WPYHAVw!KB1&K;ep_?#y6*;QDlbsW?=ME-i|cp?p9(&|^bZK;=jNzpspOB*SGRB=*wPDZ~e`Fquc)D#xRy}AzM!H z8F_Bo#uW#@w|^-0OuU$izf0ko4Tf+dlJAV#hdWzg$`#n+zM><6u*M4+J|2CSJ@{N1djdyh1F(o@D>5{?d00(U!?{b;3 zL7A;nxxQ&|ZTIr>xfkQUh^ z&5-;6h5t>{#+zwMao|%4erw!0CEQ{fSF8fFy|XupuhaAP>5tRSdflPbbwt%zfy?e| z1zmlx75Tz{z4W;zV47Y9yr>#4$@_iaK9_NS*fxxPo4$yLMCNE97dLFaA=RJ$(X9P znGEZ-TEu+&4+GZA7uZp*M4@55%yYe$B>NR3;1 zBGvdZvsZoY2eKHvtyvhAxfrBx@GGG>szqSK^t{XbD>r z&Qq`G2%1aor?p_tSwE>VZ=?UF=w_&yL>nLzidP3w#wp=Yv2;G>;txgqM3WCN622$S zvl=hmazpJLiMd>lyv^1)yOVLvqj!Nnpsp}~ZSF4LIjWX3L-t)?7<&`WZiv2X#4c-A ze>N5VIw-@m4s)g6`%WIb!_TffRAH}pQI@#k7%qJS%CTk50|xVtWA7Gte? z*MQ(iXu={jh!cB|_pdsdr(UZ-brOlSv#V_AD&n zP-*!l*Tm2$whIZ$PH(hBLpqstxMQl!BeLTMcLeDheRwPH!q%Sz9$aAGra@Zw>Q!s@ zGWp7`>B$tl2q^9*H%d{l*?&B>wM<1L1hwE`VzRj6c7vDGr4b*|%htw@4}M%Q@>4zi zjs3_e-^3tct(DQ}*H(NvJ_&xqrAbjqH+>Pb^xxNkk|&n%t}8Pjii- zsbKxRdXnt7;P;g5_BTl5v&IbM4};H*r$hEje9Vp$H!iO-I7uAFH?H($AUW^f#NR+$ zsN0f-(NQ4CjmMo^S>U|NBR4A%w7DBQ*23v~xx&{EQ1{0NsiG6cPrY0)r2SPrV0h}` zSV;}{vOdnGL0;G4Vuc`#5vzCC_`5Rste)V0ER(nSs5&YAqquM*vg+qEWTk*3h_d}D zcvyQ}pTH`}D6-IpKAjd^yqhZtXIad+x?rk;&HfbkuHP;qcC>#O8{0j!2W3;=52^mB zz6S;>bv?hi!-b4;!fTbKJOv2Mv)7A~i&Q7iU^{$qhYVnj=IUf^1FGm|$!V2VvMYAt z=+GO^X$295WrQLBmHT((&y3q``P?nHrfPlaP;+t5ML5OC>M^G4%36tf4EMD6FDrR% zT0%RK>a(&wuO#{cZ=5C{a6c%Ps>XVg|IOu!e?@_cLn5wHb&dUAdUokjnMPmW2%g$P z;yba{h`mmFn}uJyjIoLOQ>1)?=!_nRTS!Fof8uXTTn4V}as}77>hD)x}4#vHBbzqEGw3GEs-71pUUL@776#DoV-~TweXx*DGb;)~g#!z#q zT|jGwW%JTSxoaE#)6Oq5w9DA|&G;?G2XkW`(N*Cl?DQHtXNxDd9S-fLvwT?R;o#=7 z^5{f`yu5tTpw^<|7QP>AdVBkLzcG!a8k%eP^hbf|j_>TtmZa(4iR;Bl zg&F<3B}@EF=D1URL+Pc&&;fyXREt?3+dw7<+X%KCF(d$*e-I8hfgKTW}Xi@E%bMkRc&iTW`>P2riI_ z$)k#Q;1~JiC-2xfj;K~?yPnH@q|(Jxko!nwvur=cLg{QNv0B7&*wr>9Gf_UZ)QdM4 zeCv@t%629`YuK>qBKERJ@#w6U`sceh+}7?uc%I3qhfmG&{Nm8vk?WZPRB6~AJcC)= zk$;Lwv``_^$i3A3rcE}!3e#G_J-RNdD)A}h+Laq$ZJxM2tG*V?ULc`)$}X=9>%v4+ zq9~)SF5{epIaLC#a$@CrVL8lR#!b2N0pdAVNre~nAD$xEyEEV+B+_ulmQ^^h0tt-@hAN5?TPf6b(| z)CTkSQ0Z4=rx!l!jeMGQsnPg}aCPFf_HhWv{UH|CNxS_f=;Jh4Owdw}y7?%#c=|iN z#XDRWC`iWH;%uZd+*I1(x|q2b{_RVtywwL=5hM@h+hMWl)QJh{emz_sM|^nMIz5uG z-12*TnWAgLrJGr1)wCKijIUkn2l((};oXmQLE;13<@5b3=e)7;2u8a)-QsxcaKH$V z#N~oRC4+s;(ISmD60AZ}Wp&o45n?SjEaZj899Z*%`^z0W+OA*jE)G9s)S9P8_NPLk zvz+^!g}=zxdwDy0MbV(ucdi$ET&=J@d37_mMUG})H)n=7AjqFsi0wwvr;MJS*ZpZ+ z=@~^Ib2oIakL;LAb28C=xzH6mF-^z7aNU-e_Z5!fQu( zYHQ3+-dZDb+SWEhYcH~Dl5jm;MXTqRfJxXjl2$5Y(~oVxoV0omVpJUtPM!8xigyq* zcHX6F{JL>$x_SWD6}#r14ia3)f`Iu?z@b*H=_=r~p>yNOZOobmbMcy^8TZL-c{4{8 z#DjmzC}Q!urzm=d`;+bL(O+5;38;8bNJ0lA%u~3)exz?6YPl!x@9x2XLSEyWQPP0Q zorqF1wx6(H{Dv&;DZJQwDr)n#La_JbLITTmdB0{&rXfdM4@nXGkIS=LmI2TUsT}^= zz~eM)8Z*1M-3}acdU4B~nqk8dxwVN#nEbZdWgJ5nxi1(jdRH>5QYWlB>>be1D313t zV8~v7CLbXht5bx#(}dmLQlq(7#5nuK%#%6t4PVu~D|WxM5Z3NaTZlO`gW|$W-Vas& z=?}eEBoE4;^`7sevC!I`o+qU4**MnVcwv%S(|nJD+7rh8^~oYrAm+l0o-$H%!XWK3 zCdKE9H>+GdZsl9ISdM+WEMme*#`9dKfXXXyo%i>dg%_&28fjhJ4~otrj}?{vAgZ#e zi6-WCvV+Xmk9t1b+RtnHf`g&%l-F@VXE4>{ArBCX}_9MaE7CtdSC_M4g-yC>gY+Bj0^XUKAQ zds5yJC2X;}PtVdc;^f+fzS4Fk7k-hNBmPt_S#ne*tJ@~M=jbic@CkN-^5llQW2{ql zx3mWKz4UvB2~9fdGT~(ljKsKDOV!f`6|3zYsJ?Pp8nq{{g+}IlS7@;g+=lj^>Xj%c znLc>k{-#$j>(xh5f}l~4jyBorL&qKZ+T+C-8H;El0gujC=LneXg-Jq~zjL1R!n%Ky zDc~{k#*E;qd3fQzG6nUI3A%O@kZr=CMcsO7yXiQ|BX)rX{abn~B5r$i^sY`BI?JYM ztfy3OfZtG(DjS}{NTb4_hMKss>1CmFY}T+s6J=X$Vv&tD?k#3L`Gm5P<3WmHne=B~lbci6Y>SUnQ}|eI8f~S}7N+Bmu9Au@l!~8R=3R@6 z7%@tu134&Ok8~02$pGB-L}}`qx2Onin>z92Ytl!THBScn;>D^%Dct1y4$15M_AM1j z2$JAbHen|DQ2EbR)&phZlk*rwtVXOS zb`^|u5o+TU>s0m}D0K@-`G50Tk=P6wvr`?BTX&eOu$6Qp2EM9A`tk{u2}t6g;o%JL zt5NhL8EWYCgu>M~6De*V-I7V#${!2suZIRi;WimU-4J(utzk$Cd5v!3gK~juy^7oR;Jqp~~UOlrFNc?o{>&b1Djqt@8$Z_K( zt}T3w2H>#=oC$-urLAN*{^-cH z_0Pn~drPtGvdCx=#&|d;`gtQCKXZxN{eTww5$@8M^SD`G%R6Vx7<;vK(~lIu zt5AA}EAMf4vs55rBjnd}z9pzbWgk?@9bY`TqfTZuW|^UDE`mP)^$BA(AAY<;{|pYB zM$(QS%-2%-$%Loyz!#!>1cjWnO};Kr9hdkqX7m;R+Yp;5fFaUlQ0SyMHIV4bEn*}M z5hWYLWruFYZEn-miemTt3;-KkWYNbEs`#MDL!mmd7B>b()1ZxUGSc0nb?kqk zN4i=zTlf`!z=cntaC+hUR#+CpAX!C38yC47SM91S>MB27y;xr2Pm4rSocw}g`s4K1`ym0g-H(R~w;-?L}?Oz0i>Ny!!L$y~k2-Jv?VZMuo zvxPa%H$YR!nM;!TH9Z~QctBVnsh}ih4lg)JMdB8$-!fZRt%_x)#lhT;sG4 zE9ed(-l*zmR3a^Fl<6pTs#a%E62_i{%n!z)jlA@I#Kh}ysdiEbZZNRpLoG%=#ZF=S zi_9ABG%UGV2KmplvI@5X9a}wn+e?G(TB&-ecrIwZ>M=#^RC3?rT-7I6&cEmGg+Afz zwTs#A4O6tvU5>NXwrn{Kq5thwh6d!f?93@aE$$7D7?SC^zt4F=W~IP#L%0QZ7|#+;t5a+BmA_C3)@Wa<$}I-AB;m8lqj1vTVVjNwg(EttnrHs2R)X(z zj9?uhSUO_`yY z*ceH54yC{tOBggjjI!L?l)Dj23%1zHEiimvTdk?uY;|A0KGN|+9si~6BME-mpu&|8 z$FVbaMQf?9f2da}J82D+^gP-T_51Viq3xi7nIH8cZcD#h{$a&c?Cghady z;rtTvy}xHgSgySF-4TCCd#!|jP8eZ16a0$&FrQ}l0s&`E_bGFb#b)z5SBn+xhg_q$ zLsinfQL#U5xryX2snb@THYg_IA5T|Ld@hY0wHmy2+j!~w;c@Q*EY7!LK&q03f#H^) zsb@2l!LZc$n$p<=12-Om0rQFHP1^_T)o~`cu(8<@EC_OyXSAN9YJA|C;JXU{=`u2G zi<#bf4YS&G&rcoD0-g${mKLCQB<05jk$s>O%n@$ty_PMscWiv>yhjAF7+er_KRJqk zrOa;RqaPPk%`EK-s?0vT03|SR@OeWWbnJSLP}&#p&V|w+@JE?#XkWCzL_tTlymut? z(+^TS%O^tKdnBA7A3?grF>nI7&Id{TjqOweDicykXt+`iH9$HJP%$ zO*jxT?=JR&5&LJ;D?3~I?@tcT$mn^`yXfgZ;?m4ziVt|UqiSLh75b^6fPg7FH*#Fx!KO;BN{bo zqCi#f?XNhPgvtk^2Z~M`w|uymk9LhMm)<`W<#wwtLZbh-i^Jb+72%8K=gVg=H(X4! z3D=G*_C1>5S*@5)R)qvqJ#&MV6mRZmmYF3xI9#)`QmFec?V-EYe@~&s!;;T)aJ+;7buH0 z62OLZk;HZPBSBj!=Ac%)UK)AEl3AE;OMl)h8crwm>a@@B0yDCb`|IQ3A3xp(=~5AH z@%eTv;;D{ILTYNn_I!^zR5^38W?g};g0hQOzjZ1SzR({Gt*Jni{pvbI`iFOIQ9;7K zcz&^d#dGp@qh%1_K7BvBv_2?Lv!l^TO+>ncV1wP0LdqV(^!6W+^?QnO4JGrGMRO1( z*@NjD>`3t_$Vny?eT~RF58CzAXFf?Y{SQ}vnR88Cmvlo<-3pW``>ITVVfXmq5bAYx&aSfe%O0&>n6Q(C54Onl=Lv>Te z1Zbo{U%Fw!ZH&wni~f`Pv7Pn=4LCMOyGu{nL#`Zxxe%x2LAFA5c2N=cH6v4(uflY9 zZY<`&)@RQfk-6X-^Q8IcCnVah{CZGt3%y^d!%-OU9fm&@Sl(?$iHiM266QZT1b$E273Jn{7 zu?nGnE`1++Eik7v2Nxbi|6cMX^VF2toNZfMGd~@2aL^)n$cRt6bn1P1$Xh!1uX-A> zgk5P~d*Y!{OpgNyAgWatQ9Bb-H@{9z{f4VYp>A^hoL^25nsf8^>?J3giHVA$hdlrO z$h&Y+eqJ--GY0DE{spA9FZ5HS7jgZ&Y%X-DrgdIL6HP{b(PqKK=Ds^sxpUo?8)>i~ zu{NP|B}WlEuw-HG^S_Wq+&wLyh12xhAt4&bDDik|eC?|rO-Jd=qR+T$L4|caWje4? zj85^gG@-M&@%W?l>9mj@9%yZwh3kqbm&a*?%W4K(e)h+a7}}BauK)TdqHkV)k@nLu zfcAIPPydw>i^`nRF(OhvoW`^9mQU0=&+6^}?-^`LHzm1hp*i5`XEFKllp5F2$}|g+ z<|7j|2S#inZYmvEc7$4y-uQZc%SU0ZVzWcW^_Ysg#p!`R0}_2MOpd{(A(~!TEXZ-Pq-(h$85h+~zg_1MaDzSEnoQ_Aw!(G6&g_jPUC$yY zf%=wdp}IXS02s}1{DUW6Z+<^KpK?=E^wk1LFviLJsmr-p#Cw+M*4Y6TC%A2P=Q+;=AxBXgS}q318H#pP!DP_f=1FFhO5q z&eX@!zE?5+Q{m#lzAlz2A(ihZ zdymf-uCIoeamz|jyr87vu-PloJ_`NxG8Xqq^&QclHkgv{{8d*Lf-82AyO+A7l3LXN zNMIzV==ZfR%2E(kZ3#>@J{g#g0^K|GOn^@2)2gvfl@pab<;P>`ks95-pmFtzM7i&4 z#ppq%hGud-bx)4ee0+Q4pw{DI>NGDT3ZH28F{Bn?EKSnh&;uz+~kR_78 zw-Z>FLh2Ko(_ra4fcy5^HkW18{#mFcX_&$c0~L>I(`B|axzK(jzC9mV+g&RzauoKQ zD4k+x?YK~pX)$cJu#p?fY{|^Ia8+O1Mq6fCVDvO8k2fs>_Jc0V9V_kQlE?0y{-9QW zG$Dwb{Fj(Wv|6L!$FH%v_441gUc=w~)yb)%4-!{K+KgKXweL>!+n3HFTTAuWokfYX z8v+d$NzoMod?`r(gJi}Za^8^R4k=I%ecb)YQH(c7vuZiJ@5H58;ya-l@vAEg$DLMc z>+9m{LfQP}Iudoz0B)lgS$h%eGr3kzT=`&Te#YgR-Om&mfs)s%KW|UG*Ck>h|Mg_V z{?ykD#O+O2}2I=(;0RAKw3A>0R(tE4%=L-7$N; z#|M{4848v9mQlEzKu^Nbr;_Ea_+t!7;C1Sk#Y-!{FI3KyN;v+@gBAZxM%iT%c`Oz4 z6R)*Pm%1s5*4ME-erQd`EEjowlQcsXLA6*p(>srfT^*1*X~@gF zwaP+Thd6L1{f@s=V7XEYYkM6D`&KZd@IKe;z`+OjySoK1wqAv+8yX7JX`!N7C2c*w z+49xK*2PEX4KwleV#HHD^WXC5ba=9%L$8hCX_pmDYkCMdRzf0TK=evMI)u}cm*v-u z9KUQWTv*MVMmbXx)AKr3H!k-SN$yPsDq;w%tk&XUpv8UMT0>cAfAeH`>qtUcHzhGO;kuDWsre}c{$Q-)y*0%pW3}^J zsP*Rw>xd`uac7u<1|fjQ2~JW%TGFlGbw`B);Ne}VDeYGT7ffX(0>!n{x4D*oyYW8K zxiD_|$G-ssC-s>-sd!SDy}#aSD4v;QUeUJyHE<6qUknO3HpL0-x8cf??h8x|6>g|| z+q;R6rJ4^Vu_7bM*Ef`fuYNsTSibsRs@O$bdgBf1o10NgllXI7h{-oX?Q2ckZdcJq z>o;!WnWTWNb>iU*m$3UpDifn*2!k}8_x1tJxg>$JQ(HTRj|a7hCH0$b|IRV%Pu}{( zwv2n{dCDGqOW!ORqs)b`V)b4B*^HGUb6pQL}7h zM6}hWy4th!uLSYiQrU6dgHqoy#__i(Z9v~cySW@`*?%xj(XjuZ5r1c^uMkfu$x-6Y zg@n(e2ep6QfNOPmRb>Whr*Zi4KDa4>3UnlY!(TOgFg!dgaW-P;DGt8E_Q3Q@3yz+E zUC(C9@ecGsitWYNank&WcBiF^1(BZ6@$CvCpk+6PECi<%Rj)lqG>m2kAFN; zMl3WqmQ2`szbZZB1X!~uu@(dOBMJ91M9QRt@3yTv`ZX(_sbmsr;oN#noi^*cLxbkV ziB0531-|GF-#kNq+xNg)KX85O#9TOGnBAx#>t$q4MFsVy8(TwhL+6<{aZHJ=xze*UmWL_{Mxr z)N(#}?68-j@sdvGiGy|JLn;V34?)1HKwCK+wNS=`4%p=NTx|qprv+ncGk3a5LcfAt zB{u8O0(hAzi6&^l>ad`$Wf8qbdmZTyBX}_1%Bo6owv($y1HRZtgTnEJ?49$V{v@Z6 zAphX#bH(l0r5y1v z0Al`F+mjmr4N3POksbzyWJ7tV-WP33oL9&y3f%i5X zC>!3u@*J0ygK`G6E3B*2^t5d(?vJvr%x~@#5qYM2e_klnyBv(6`;6rL zlU}8onMZR6lXof6UOX1MtmSM`zCsjy=Z1ri_2#w)THrq4@JpmTiHT9+YU^Am9#*ue zvLnxqL4Y3WSY0Eu=jE9h{8Jx8-par80vAVxh2V9X^JG)&CmMY4v(2X3%`UOqzwNWZ zU2nss9TSE))Q+yaP%G{1EJoe!K-=g)7AZW0*RH3upy^3n>TdlJP4MVlSQF=DM_wn= zmbh0|ru4Iror(s@im&^?MF&7eoLye8saCi9hlC;I*F8Q`t*4ibj9=jH|CEhYXs^m3AZvdE#6T>(9c|r@De%`KSAv=(P-_pFf`WVn z9?4^}Y@=x!z?}O3nsyi(*gIWR5299^d~k`(Iib5RLj!6J71ZQ9GP5EW$P51_I9k? zW)L0GzaRmPB9uiFDgcQa@SE8f5dZPTox3k`Rg|CoW4F$ZWuEs9{+B)3Jt z;l~bX{sW{1ARmaK-}Q#bI;~-QdXYM0)P-@7sIT#Q80m)aFG=+?4J7pj@qsT0|k3Ka(i{-rp(ix9}D4(wBMLzw+mfQ zk2{LL#`1Cfe6kdK|4SWr;gSRUr7s*Ix3 zc5;~&45GK&bg#{to&{e|HpfTTSMG<^biVWIcfEfLF`jKc(Gb^jqZTI8xvcGuBXFNg zn<#%2T~0;^{kXXUQ;;kDDT1V0u7gcKk#t!#bG_N8fb#rW;yea6R4ZG$JXxz%s?F{- zVIRTl!4ReC6Z|S}^96w}_`ahTyJdBAef_z-)%^e~V|#ysUzxQMa2-c*TCvx%>16ef zho;m=F=@SC|XCL+Rk zQRK7kj{Ro4#|$>&{lyj7xqo_w7C7<_4uwA|K+vq)`4*SGLukG28#D~G%M@T2Jg|?_ z7BtVo0qh5rkIWJEzTiv=of__yTa%fgpv!C z^PB78yrDrS{UE@UHwpgH@3R%9w)6LEe$3>&lb(>yETJCcXX)gY|A594m~;h!yyBL^V1->6f%r_V*RxxiA`i*#P4 zjakl843pJa94<`#?8oC>TDOJg1h(tu&#^Ud8bAC>M^7grg0jkG3kC+5iWCrsQ&p}8 z{>~fP=|7fI{Oo%lI)aJL|HUZ0d{3qwd$6kjz>jYDjKvdP0HS#LLg~uU;3wf0H*X`u89A+)9E#0J8c1u35w^2YX&=vtGy;;}X_l<dh<;|9 zU+bUHgBcAKFcA(+I}l)+WoNBpA~fu6r!ho^-VT+$G^vSx!2 zdGK#ekQxp`HkO@6Jy9Ew`sYq;JEAdkP}^mF>Kx>>%~n$yT(3WjRLN4uC`tlddxB}2 z20e0q=F0Nb(aYC-_NE{O&*3F2H5g#v4TA;vY9Q1*q>yIK=Qa~NvfPqQlDWKj2jm~M z-}rt9{g)^c!Za=2+BfX~3$tr^F+&(x>ROnmo5T$ST-nFmI^YGUo*yudE5ugK*m3bm(Ts?adZuww{gppro_d% z4s@6SW#SP!Q(Vfh3Qa-?D`Ev*5`O@Fk1RC8XUdeNiZ5^F*R2#;N5m@Sq*hl z+9a@jf&vW7+9tf11*bn8wQ{X;~@n{Pw&TY};mFxBte%jyG>!F)XZ)j}hI0c>-KTI_anv2PnG?S|Z|h zicjUq-tbcb?bo%f*CK9Y?S3b9-7xvdP$B!~o^7{PYe7PwHU@HG{77;KZ;AE*Kf7p0(-xTJ*ZnvDFsrA2lMcDDXOAz*)=8qsjKRKwc z(LO^Jp0_{~TKYGwHT#$@+>a;TG$hr1_F+W51%A+o%T3C`73>nby52A%;R!ISH~omNQ=h{*o6l;iGNCGh+wEI zXzfJ6K@O_!iAO>qL&II84%Mus$ zuqbG|B0|8xz~CkqSDAc)NidWm366+xPtn>zyQ(slZCY3Jj-*U~w;-SA*35<&XXlvDrMs)WP&U$f{AtR95X!mRj4!KEB+^&FtvE z^0f4f8;|-}5&+7rYpxk*Gm#!Os=|g?l!b{u-rQM_0fP>=$T^ss2B2S+^o39=* z3vROj zvM$2SGIj$IaYXh}xz#Dx8?GmXD@)@?X&E>VX_;NqcVzyJ^RT1EsF%VGAM!6;PcU`M)miA%aYeH+k?{KEl4*`w{L8~v&zFRZQPB^NJ-AT{;HYUb1%mCWooI1l(4Oz6A_zT{TEy2m08bY!M>k;<6#--N zx+7VAbynw;`5&6-qFfxrx%zU1_*(0o(0h*sfvg8$B3<;aPZk1#OR{%EnS}6jLCbfl zAT`TvcFl3n(bb*XJ9>3>jUqb(NR1N-$l|O`N28Ymn0|mwTq$cG{wpUo>Je>ni~lPp zc1uSyZt$dV)BGzDwZHBjfZ#MNF&>_? z1xwpJ!Hz|`TnaE%Y(x0b>IrDWl~{d1aK=YlGSm0U>juW_;|Jluq%q|D$pDdO@<%#- zDkk;NrrdGo9P=2t+Db_AslB zQ+aOnGOD^)9BdxC$5^Cfkrjui4fneDuMmLcR-c~M_yWHk6!)w5_87HJg5tsXLmOy1UNlu^fUXC zDu7Q!)S1AcqW$n;B(0ez2H2RrF1-n~pD-X#x?d$6(~zOOKx;}?P7VhU6EI|VAk}>q zoxiH~iS^k((c{9-Uf<8h_L*Cs%xQ5e$T_A8*(o98u5t4^e*@Q>WwMtMMORfrd6#=_ zY<6E9lc1f-Kd3_S=5v5bx0mW6sYzI*U$wT%`s{uc1sjR)`kmfo_B(vEhVD*OR1B`7 zdi(o3G(;yu!kNAB1DZHcU>R@+*dS?2pv@Gwi6!ghB|hbU;R|VndQ5Vn7AR3(_r<|( z4uCt+@nckCP_kY-tp*`HxGe0pCC!W|!q4n7d^?c*uLKV?PS|fm)AEHG+xxdW0-aI_ zYpE7;VF9PLId%nX9E}B6d6SBu5u7#Nfs2#ri{{T21(|;IU~_N$G2n4YfjtXn_rb|K z_N8r5laParx>xiMOOSv98ib;!m+@T3e>i=%Fj^h+u92_G2Wh0L?A4nnD4R{rwsAf| zsIO{dU!NKzz__Fv+)XH0K*}OI<;j75onV9OBd`?x@$%RkpqHt5T31?1V(xz?3=A(6 zXhS{$tG(yMB4J$lR~64p=Qs2_r3tb{@Z^Wd@j25P@+GRlnF5_R7qn3WBk+I0hNk%; zNYB+l_vvr=k$aq1;UAv%aV(kJXu8K#B#`BtUoC6t23G@UgrxJpwRZi`l8FJ^UTcod z)kV|1mb1h8!D{A4uh_C47G0%(uVj#0cgu&?!C6+H%|!F?U98bJ_Tr-7pBUmZ6dOZI(y7_cCr zYP+qCXE9he4uRxRZ(_odkN34%uIs_lHZ_OWF^KTQMPFVv^}w6Orz3qh>h_;XwhVM- zznOiv@p!J%uJ_UaxA9xkZ!i#yCvFnyl?0ZdVnM)m=2#%FJPWqHQnUQhQdMmOBC1Y@ zTaT7eUf8&}HQwhyWSJhq4;IT7@+Dt7FhyS5@9D;A#MzJWLZX;tJ3_x{IH2pP*5<3! ze^T}1+sWS%sM*5*=3-OGb_+9omI?_8nE{&Cz;H3Z>i4+i<>#lTy90N94*7w^^{lQZ z9IVLSo<|~We1N|K#ucpmfn2nNJ!erMnKg$bfeR6Eu>i<94x5_7)C#bPao@>8xg@q9 z_k-?Jv!U`XIi-ZB`ZTxc_IWI6^rD}PF=TLad$gl@oKJA_aPw1#$q)j%(*!xZ)XdDW zXx9RkbG8HZIWFdwmHafPll9t$J3U1P#qXtAgsg}%{En^fXG*6t+`C*wLGE_tPsubn zkq-Ykl}_(N4f@2OIla$%33gsvDyW><-2K+t(=j$j2f)evQCcXW7Su?gTHv4T@%TM( zdQ*Axz^C0)RSg`rQF{6EWs*krn1l>@eJp6NyVYAY=oQDw-8_Ta!u6s>p%aLx%W73I z@kbV&f2Id6#uUIl{z?Yd1N6$}3r`3#lvYi}YyXZHaGLAnC^v%8_62}R!*Q|n*^%y` zQztb+q#DX<6o8~8+rwjf?bEiQk1iLazg3BoQZATdPVvOO5dL>Em!1YK0M!*GScs?- zm-zWLZG+y2jle)sj}09N{o?=0clQ?Bb0KvzhF{{#oLv|*&Fi9BIk!)+ z0cRfWY6g5NLPcuj__{u*CezW&WUhhK9E?beeUdOZkkU>Wum8O2%=Wt^bms1ZPM=;( zy?}rrkPxE3X>KV2+8@GpQowc_*Of*T{G36lwFwM zL$WjzBGNC;8*<%LVveLhX?VBx8r8J_Ze%}-5+wpELw*ZPFv%->@9>uP1BhI01IT3m zhZDDX=64mj ze&a+ln>{lR$XSd~U$Svo2L}tJ#hIs(z3aVHNboZ&;@oXLxG}pGXxy1B13zEB+u8>v z_93PEq)ny9NZsSme^%3s^^8ETXpxSWw!i5J)rh_ImkBRb{4<`QS_Uj`S(ixL1Q2Cn zaLNeQy|e)0O$M-^LUr1h`Xh6?qlTq%#wDh^_mH}ga08IYjM<^b){LAnK&iCX|c64=L& zoxKShXfA7Kry^`vgC*`^#bYp53UmxXCImjFibE|zs;hvi#}F-v|T{F7hy?_AZ>y8OO~0W+jjPoXZLcG_zPc9}vIPGPKf&t?4FClx=(oYBYAU1b2HW@TRCjja&GAN&{TNk?|-fHN-mt4o**M?I9A-mq5^Mi<+%$syi1M zVIHXk91&kcQh;Fj4F?XLG97dCxHy?}x^NPvL{MPk0o8{spnLITZM33py=Hu7VS&?4 z)z=tQC=B5RVAntTi(tPwVYM-j`-l9zJT?vvEXO7k4Go{!#GEO8jB+Lai?8p5?`dd& z6&Tf(l@u#0EBFKiL7D{$AO{6vokMovya^47h3spyR{srY@dCiF|MLElZHi+wZ8NT$ z&BwvErFBz=%fR5uc7}vJ0&H^nV#Z(SE%L^hO|qK%O#6+e41HrYSa<+u53y>(2>rMo z(*?Rls9@$`uq`O4LG>C>@o*H_j86$x+k@>wXJ3F)ySAQQFpv<5k3)ILx8S4iO#RL6 zHb8sL(l=||Lno0y{{+Ouw#ujd87iEy#LfB#<@MCjAqOfmVL<5n6}_+;Qk^}AraG7Z z>39XaZ@HO&j^5+1Icu$0PDA)?GPWG!k#_G@8=aAgv5z;Kw4-87WaM9j`o4_-l=8L7 z23i2QB*Pn}ivfofNGFK7iZg+V*JS;6`WI)&1I%jbBdpsQ!)$B!Rf zaZF1xUVAqU%gkclGcKj;(B3u*0Ky$L#V&})6%iQ7y_Fvr5TM}h39yl?Srli=i`jv< zuT@)n6m58C@z%N|jV#Fbn%J64PzT(8ENvOcCENPky!9MzbzI>o=|@O@uR|dK@BY7Tl|S5 zH8eH5-;24#zkM6|>C>H$536uPLqk1*EKFLXBiL)++uJMa<0DxLM{@g|A1$H(sH}{; zzxE3Y_!vM;$0`pm*xuf*TP;IY7Pu+=yP*%y)aIKivfVL4rsdw z0Mz<(ipdpHQWYB;Zsk};E^8)5#f@tg@db2QG>dO$qONklEzM+-aXg5AVTI-`#AGtg z7P215YHx)9GaU&s^7x43+Ndakf&s@&-y0ozNOyvI%-VrwO8=L;*4su(N;tSAj5pn^ zeG|v=U^LN6Ykfq-#OpP?0DAy?cPcF-M_0scf8L$|^2tW!78t+^}J07QyLVV6pBapSp{1~h;rD&R2_-ZNI zUjY?diNSJus<=ttgGr+*D`FA*F*GDa{QpeBoNa;E)W~qYdFvKsgR;rV zQ>YfbunpBkDJvK)WVU&kGO+URTwTSpsHjigX-YX|vBX|mPI}E8)Cq}`d^uCOFxb7* zyKI6O%~Q>~DT-$yWwm@1?3A6n2%rFk3u8i{{0R>PpFVv$*OUH`@)J)iY8ogHrQN0^ zV_}H{2N^;;LSug$i1@7FY2@j0Nl&#nPD`iQqHCM3$1mR9kdk7;BV~3y8oPDtRzgAo z24qeUCq_n7ODlZnxvMoVd!O|F1!~AX=L7l+=pZ$SG}`+5h=pAW z@&s4ON1fA4>2d)c{j-IiWiU7uE<$D+0m!b^B1soQ8vFIBry2s2C9D9^@ffc=lI59| zl^Rqtb73FDM(c$SrOh4sZM?7D7fLFimLwn`c-L54ujIqDI#yr#0?{5yh%qYh5U4Ia z0B$YWH0adc_7xUd>Zqj${CgIdT z9;V}GiAh@eE(r8pbuZ@mfvo^77(7sJd9Ie!j(zMB*aRO<{;mdMUQxmT;w1sHQ+K`g zV2Ikd2$=cy$8R0NI|nADT4%fp&{lpT$!3b8qBW7Fj}*8wk#SGTLAr~5o~o*du*Js1 zRfh(dIGv~*GRW|ngQsc_g~hg3ZsTf|zLR|ET=1>Luo&bOUeH&zJgtkwmxAz<@| zCrvaCgNPlOOv@ggO;6KrD_upX$|9wsdH^W<5^troGMe1s5;!@KbX7n9*wmbt0 zJlR9f(N4#Evx%0F=5{NHuy9L`+hDxm?K3y#cCCvR!yd#*x2!{9TR=M>^xdvJLgTQW z6&wC;Gp|_^-ENuPm&vSXYG3rfc&J2xm*_%5~={zJNg=c z-jNYV7J(g9_?;b#EIn^Dm9>@Q7foVEG7E+pn@Uhu#Er3Zm5F0!Te^&VOg}aW=Z3Ia zNLoG`k)uidQmr?Mb{HVsL1v!2Ey@PN7zX2JV7@+m! zFu9fhdSot~lGK(2o~eD%cV+DEY{;N)HQ$GzzLUx^KYi^3B$R-%8v2cl!rg46F`j`G zd86sT4AKhZ#m1<~@^f=p!RJ@6UM)AetpRri?jg&txZ!vBSodn0@ok#d$;oVde54=^ zbEdEL+NbUb8I1FXT(7x3+zo3WV`MCSIieZNcaOUeSG#nVFI@QVKzV zlm;@mR+@Pl%BXV^YSk3`ZpJe8l=9K;OxQJqI2e3`#9#R-(t{wOi|K=<9r~&OI;a{* zFRP79Mnln;V`*hIS2OMa@||K()Obw$hJ*iT1Ddd^s>>_HfM1v!b17@wQA;|ahjg#Um=wMl@0f}dx?8vQbLvJ}cHEKCiQ z=;*{il+-R)L+9k&y)uGl+ne5#Ebnh1<}_$U zs|f%+IcN#UoZ=Jd+{^CcOLrKuVHjY;028QZWPCES?{D^O`>QmmP@7j)12W zRM264^9!{7-8z^iRdM19xu4;NzG>M9p#{}g%JcGjC1g})y5?@ zk(8xis$lw5@q<2h1vs(*z+(cJ&+P2t(&a!mGq2nmzgJ>7K1ce-xP+$Iu%sP;o$sK8)dD?ri(lRe7yqg@t~R!>nydmj?3@(~ zz+ah}nR)TrfCsSI_Imfhuy|ZJBO;e0iHpy@KzR}iFr0#2r~@M~7HuP=(cjhR zb_FO~jVdiw8&U?@a^7&=9>%{t7S)_;$N7~%UK&6vhwA;AODH~B6-D?k}&;LnG7dnRV!D63jVjbgQ@aHJFjS!?2J?KdA zGWrSef^teXhAcGs>hYZTA<>q+?7i`ebGI++>q}QilXE`5xEz6$8gGp4o;UR^`)Z>A z1{G7vN`%B2GTsf74|efd4r(Isep4^ZUij|wL0W}7bjhB!`Q1cjzX|fmZQ^KRb zLwX^Mxa)gN)hvNxx=*a^p1Uu9dIhvsGi^#GsD}$!Upt0sSCstNm2fNY>XD<7rj@tX zT-|&mYLk*C?F;{LJRPf%6%-WINlcTL^#1+r=AMdk5sS^|&}N^P4XcAT)!>C*5wRN$ z0LAAFa1f#3NJ&6bfKF;n3*^(eXCBxMZSonC*tIZO75D^@HWHHx%~}!ND5k@Fm2bu6 z`WJtY}jbz(gPseF^EeXDd3XFEJjAoM8f{4lp z=PRJ>{a178V&RW}e+lfB(vNNbLO5EvpJFv&@}dS;<;Qw|%EaF9)QuJCbxTiUhrQsV zsHf###LLHH5wLLtosHAiL0d@HFQ?dJ8WokI&yRzQe>N{v!cMjebV2qh|Az)teZL`F zVwieOxe1o%Rsmfwy5aq+@~P(RGnZX6xb(p>YV}ct5#&zG9SxV&pjyMtlG>>#6UV0N zVX?@2;xQfi$#!IZF`s5T#WH!{j)T|bhARXnkUH%LZq(koCwcZHatDp;!Oe}K8tDpop@)L6BgeuU_Lg*obrnO<1F?01p+av3xO*P7mcK$Z)u&5dngLV z=$+}g-Nq3Vd=9lCVt%Ae10IdqHcqQTHd zOuWtg?PtX7%ES$2;VW7g-tkw3#OZ2J?HM7h`ac42ld7Spuf{f;As5KxC*=?Ad0W7$ zbIJ$PqC}8#H)wtxFE_B(5}{P`w|5{M&q&r1PnL5jXRYh`@%@kaX--^D8=869k%qC8 z<1M2XV%#=;dA#9pBj4aF7e_BCf7(7b2z{BL@B&) z1WQ!P+P~9x6@DLK@HXw$_Wtg3=Z`t%xDYbNmFU!=wvr5mG`fhVECZe3jhL9med4^r zm5m>}rJ=a17-|aqCAwt#E0Du!8UH}w9M2O7EYC~{AgO0PxxD_a=2j(}+Lr+o zEL ze;!%>(zOGm?)b4gN^{AN(1X?uGQG~$O+%+0`x`|sTyh$!Me5~ar=|SFrcQpup=X3I z$9I(tdEQoT3BH$j7zswmPs_I-SZ-E@7r127;W+va@Rk^qz{2-Oxo_ScUiUIz4;ZQ= z^Sy-~N*n)U`)fYf3Jt)B!mBKP9i^dU&i^zNTNL{c1=0BrFpS`H9m0pMpNAij&> z6cclrfk+?DU_oScoh27}n#UFnl$ktpJC=k&;lypnjXaxMlQSkwtM{ZM&@x0j45GJG zLsk#0XFsb!_DK*MG-Ns5FvLAZ4ZhC(TFq)(A#U&((lVHXMK>?xChl-=_mzv3l{NVo z&`bzeic-;xq|{V!pJhcqgy_wbJG6u=B7}UJe6MD}1^7s*b2xJHc!TNDy*|ZBWi**C zcrclAWTE}d3m(fW1F~TH4fPc8gS~Xa^J8AZs8TEWF+h-bJf3#r#inhNu}@=Dn`< zhz0AlggO!x44cvNa|YhF{}e%}`*n6^SRu_ME&f;dNBcgguL8&bs|sjUsh;@#Gy%|a zrPv-0gx#J6_;`l-0%64|rp@d&_U@$elV*g6drVrogp?@` z>!nZRL0XlG>)SWsrYGqIm!{_YYrWx(E!o>!)Qa6;Mo4|D4?Zj&?Vssn(1!$r9}@yO zO?PTk{K9!1by4s=Lqw3^TS0mg`Gi2~%AY(Koq&9X#W+D~m#coXGq3MV0F+qUuiloC z0!`Xgl%AO&q~BZjCAmZ~0PukCL|jHFID#sAD*7rVb=MF$k+63>R?E7EdZghXxp9mv z&&qa9i(e2#BJkdJX#OWb%|}L|yABB zWrKO_ua-?Zy_m+2s(u#X2BY+3B4|`s^NM7+z|^Y_rUK^Lhgj>l6;koH3mE%?J7_r5VFT?nrow z?WPbjuk{^3pvzyg$3n(DwsHA1Y}@s4Z-es!^!3vT4v^G5q#{CV>QL5WiSU$msz`#( zmLBu$>{qEukJdK%rf@Tr{n51$G~jeMn~kDMy~@igq&y7yl!yt+cjgMuJ&w4TJ%!?p8O&;8KUj5>0DZV0Ze!56gj?boLBX+l3pC(b z5ii5JOd`kEuU?(*8%~}*_E3(G%M`*TGu|iw4siA+{ejlyv`$SBgmMsynFoFWPdGz= z=6f^W3{uj;ZW0d4US*Z~mb?iZiXd58Q>?f20bH+FU$5P4KI1~qn5OIj4bD_X@9VfS zXu1@aF*}rp<##rp0EPw$k_JDNeCxBIpTA+q#h4B75zlr2bV zgQM?T(gzB3RTMMdHB1If1({F$ueM$Pf=L0u|Lgx)&p~XFqIrKdWGqCi`(`3#fl2i4 zbQ}r>SLRZZEC2jN4emOQ3Ib^}6b$Z5RtUg|ysj0U=Nm=ze5gD%Ec;qo|AXtFqo53l z8@w&xCBA0@w&~FZQnj8s1RP5MTnrLN4 z4hd5|YC-1{Kk?&lSqF|KOHXD55XbPw07n`JYmd!$e5~QIhus~p2rxy1M;YnqbASf= z4^};A4p%;AE#uo1@L3Neds!#L2qJdnCSJAZQZabOq9YZ}` zxgy#z!0p2ZTqG$V!1}r2@@yxjbyeS)cYJ%FymmtX9(G}7#Ow7BBG=pfqrs!4oIv+E z%7HvE=flBiZd%^KX1qKtCL-^!VNP$Jv!eCbP5O)5K{E4TQk`HgULoTMRV1wd_FE_K zO<@y28(*`)vQU``dMRn?YQT)W{HX!T6(P3`6ToQvqoh-08OlBlZzk1Ui9$4g(Wy(f zCpmq{MTG2rx`oDiAiLF6`IQoa^|PRy4}8@8tY6A>Bg_&(`2hO)N<>qHjz%URF%d{9S@qi^AlnQT|q1Jbsc8=-gkawbb3uGU;ChNDl2Hgl`Ez$Lx`2%MbWxP-13 zTlG~=U#i}etA9H`E86daITv|&{d)U1ujbo5G>jtqw^1#d|#&I&x?f zq8Kr*l%DCA_(8oKFOS**%7{Z8$Eyel;8g^~gr4uxoSmQ8b3d<4GzZP@;VP40L{03P zJ;Lc@4^+&H=gx&3EeK+(dk3>?LY8dG_&YZH-gclR9&`2R{6eU*YmUa`Qj-n5jn1|^ z_|-BsNZ&vr5Oyt$G4x!qQY|%y5tX4Qr^8lx}9} zpiLw#KC6AdBgZKSzilbN?#Cl9m~thu6bAt4%r9QN;B@^hZ(XNH2xzaFYI7Ld)#7ua z7l7FgDCy;Z3o3YcFw|PWYG>**6kMiJ0uO<^>7Buy<1){m(>;6k2w)DwZ9f6F#=?RD zC|U4+84C-GnoTb9xS3Wz0Vp!7Abtl(^!rNH3Sin$AXZqJDAP!V)uBRE2SE>a7vM_I z^BDnVyS;Q^^z%YOo_A-05RHE=va};5gKjE0Zoa3@N#P2!af&dHm?k=6sX<0f^+#bD zec`2416n8U5$Pi5tyXNqSt1PRMKa6{O8#;PaoLD)r3~b0FqC}i38(gmi+S7`OE$l) zxV#d5(qj?|-oCb-hS&TCzmDH$KFzVbE+c9A9y*hG6F@)Z61ARpG`W%{a=)+6w#hO= zThnoWp{#;fsvmfb58+ba#n_~%MEo#F2q+1ZGIDvew0Mo3-T3F{q1u(ZAJyUy3$E`L zTyVs>O2)lVT?)9v1sr1(`?*>)7*_Jl_3^X&OY_2_qNpkd0Dyx9^afa0?ki(VLICB$ zpe>h0wTQ=xA9?-aiCA7i5R42Gy;yJqb>hn^4xKL$AYnq(eU5T)U{r0+!|f)cIou6- z<QFX@_Sol)rG&$0`u$-!5o{+p z@aF=r&7t2%nK%+`~=F5Y2Qx8e?N&+c&GA*M4` zg@^o7_S`6Pkf5V63&zQEvw0hH+?gM*2??M7p-pG_!aB|NH?9Ni@ z|F580kJ4UK$_I<8k=Ma6CIb+K0DMg<7-dwDPB*=ybho{b}6RJP;U7b?1yXDtcgFhI>rIChX)V4 z+vg{I*cu4t{fpsRdBo!?&oVG4@C;k(=?9Ouu_SJ`(0LZK77~--?dV3{N`KnALGtl zq4N+@Gi)+ZOj)8$HS_W}RRbhrt6YbgP3IR;la{h`BMEBd97I1`A5NcTGWU1WpMa~t z#l}~HO)ks*!Rx=v%|Gh^F)Rx9pHlyH_lbRA1Iw2F2p#v{?er^IS^f_1U(Rxz`^T#X zJOR;%hM$hur^lzCZ2%8p0z{zhYH7NN+$qf%zMR&-ynW63%dnr2uzVb(9PcIb#R(OZ zHzKD-+mHV3HpAiBv@w{W_jQtM*deiSd<$UDjL+sCfz1pKhCb<3!#8^!5O)oO)(J^~ zy#pY~IPj`yCao#(&KD-=05rLB%w}tCh6mt6z5gU@!?5dz`-}E9A%4f&D)HpZ%vf#X z0IgkKL%#&BV<;^oHWpKdmDmMKOU{Ix zjO@pVM*hpu&FQt3Yf4IkoG#AEIIt<;9CvtBCCJU9NjJaM3@N(hm`uSz9o>aL9NOA@HAChtS!Lu+PIT9oc{3WSW?mk?l}77 z3)pxA9(E$2eqcg>34P(9SlL>QqW>1r^OHoW8LRHB?sA%hqX7KKf%E~!3`eoU=AuKb z6Rz7fm|Pbx32|;49%*z9`y{1OcKcG4o?cqC9+3P{2S^puLsQ-}sSqPfw6wG^(pnf8 zWRClNc<~s(b1-qq@87>8`@|m>?n$RW7|VLJgA!a=7?bhU;`M8oPQtl4A_=bUpVA)IeJL^#N~8MNFSxv5N;sX z%*>{1d6S2|4lAu_c%UoOG)8v|af{dkJ8SlrjEb1;Q5B~qjUWx_jmiPR&%IOv$TqHS z=J>=C!nNa=P7y&Wt;VZnV^sTj~e0kC=P7>bGlW7$}eUmzkOfdi6PSh&|W z2!a)9BK{eE@e7{#08I!YU7?P}xF-UFjx0i9Q7Eqqfpfc(lrliPeNpl$6x$+3F6|U= zk@#W)L&xD&%hj?#?lcAG)z>F`=xSnOqCI^RR2kxMB+V@I+>EB51~<9Wck}|p9H#sF zSw^J6$^dKap=n?Ezry-r^KU?eTeWRwStJi5$3eue`Lr_a~(AFCG*;=et>X z+dmrfz<6>te1O)l6NpLZQFN;0qGi;GnWOFD)P#~yL(L=hDbDF&k(U|MJR z4Vx}Bx+wF1u~jfIFlatN!t+Sz{)T|Z_n5FJ9RK#BRYN^R+OS{1AWmU0!gRurYv$}1 z7im0&Vc+8LSqrJ%Pc>F;-GvWt*apZwb%dDn(GG8)VHj|G$H#bInj3R~Legb`ei{9XFCn~y{BJmrNUu?NAj zGhhCO&EYbJTy7b*lxp%$v$x3GiSOf5T)=oP!z&B}Iq*EPSfF8b<0`Onpl%ppu<1^mZL~ zz!_u{I>140au2e7xixDELBWuMjHIR#pIPKwBY}tasWU8nL8-xpg(&E7wo(PJ9L<62 zi$g*r#1<~cKx<}#*fnkx3Ug~})V~@}#)&{JEQ6{<6V} zZM9DKnR-`OS5ce8JH2YRZ!jW9{Na{o$$K3O>HzsaEK4ajn?*EHErd zfE9G0YeXznB(lIx)Xdr71(XR+;7<7%k4ruc0bemW?BSktRl-h@f5s6Ov=Y;rL(fk) zwdLQ@gNqLtJuC}%Kkr_vC&M^88i_NL-$pg2qRc944!n8=b9`7wmn3sjELKSWk zVLsHzWynq9@sIbBakK`?KzS9FzV9I@Jyg3?;IHv*zmwzi${qK=iLETTE~>tu)di+g z0Y#_{ej_mObo;;@Jpaz+4_9d4|&?hld)%$>-nlSf}-T2 zHE$Ih716UBBEfflq_!$fv9VNWZXvp&)R0UVb}vG^el)N_sZ4&Oh1BhR-3tk0bBSCh z=(gp~WxUV_^%M(o=DKVJCWt+RdD0Ke$e)G{&CNBQza$b-#h1o`KRpYyyHkv34$>S;T zqO2$UQccEOHm&pbE$ehm3GDagB+9v{lW{QcQ-hp>Txx z-T221%<)iD&*Qq!*(FEu=Wz)_@dD4}+;&g+?p_DFoi+Lfktf*LU`^C7%@f1IVebRS zo-$~@!BeX$W-1u*(i_!b9w0gCu~U7 z`8TUAi=7DX#fN4f7^h}53F-|jsNqFTl)pk%f)vC^%R%QuR{JE`?7Pz1MPKe4WJ6}*qL&P)5?FY zNv?94aNPOkeS62QZbG5y5vEGnIHjcR&?)DKEY6hp7H%qDh z7cvY=(If)Q^!Xz{gx$zz!)Bpiw`8h$`Dov&U*-58v{LdvH4)0yBskNT8NG$Ak4MEr z<-&q}gSB?L>okMXH*@t(rfs;CN}}zjK;L1;RIc~g(PioK@{*B46s`jRCVFBy&kMc- zJ+ndiw3@!Yc}7J|-JE{2F`FTYT9E73`c>b#Rad442WU+%mWn`GX`1e95Rv~pK2002 z$G_`yAqN7#p@q(n^U#a5xK18Sl|?m~VB#+Nceq@fZ&z(NI$=p$(`00t!XkO@($%&U zU{bbW;|&A)hiwP|67Zo3b2|d8!|On-m;S!QR-VbQ1+oZInA#rJ%z zw%SyH|^l>J$l>D;P%%ETYw;*)fpTV|&{?~cX(3t3f9nPmesuuHxm0rW-8oI+hgxE_to zgAE2TPdK3S{9Ij0iDg;a)g6K>)ygTMWUaY(66?|-rrieQ*{ z=IuRU;EVZ)uPv@6)3?%hs#>oW_k~Ip_|P==QyI>RfZt*HKMmU=V)am9KT-8D>7PIZ zCsyu!Dbo|FEUZ0KdX-}nS5y#noH})YCb={T81W(3P1B$Cn>Zi8D~a_AILO+cuhDXW z)xY|}4U&kf?qx&Jgh-32;s^VTM*1%N@cymeW7{(i!P-C06?R$aaeK=)7bDsvt~Jbb zbZFq#cHw|PaDhyIZ}TQjdF$dg>5(_JKbQJJAE`Xuvp`#h$9?oX^eE83h9?tnF^p&aIc|n>$2Vz@ zwN_BR{`S7SEGuIF+(jw5omhs(EUXoKV7$DzPctbpO7Z?6J2F(^2u8ISl(lK@3VIxB zU9Dx?ktiCHpYg!*^wLH9R}{zCzs)!2o6|*$mx+t4=7Y{It^J*sMK1mwGsVYr+tV8h zTv$jYKMRTnP&*#wTLx%{9hyEx#5@j-NMF_RvgVvK5w!C3wWkIR29vN$=gS6nxIqxQ z?5e<+qbi)*+DUBlq&RQ6p%2mAZuoWKAZx0Yr|(EJsRN2PMn+{~@rV7DR{(iDtl}{0 zRerx=YOyO{rnpPY6!S9ocm>HoV9U2mF>m?b;MA0~ILqDn4{xM15_0Rm)-N95#9@6O zLaFTRQz@(yWVBS`b=2XCfK4{#SlRmXM*op~k5yGk4DC}BV}i3^&IwH^Rqf;rqKW7# z95|q*&o!Q=uDljq{Qlv5UzB~|#EANV==r(njP@*|g&Lv&@x|jxWpiGxC`-HAv2Led zz2{cxnf#j!HW~FcuD=Q$UV6T-WlXol=d0kVZ2Z!g7yK0+^Q%w?)YXAGa(QFUoxsBx z(skY8+-h9aHn1Q%xEUBuW3VvT&ZG80&3@dcSkm^_v5bSxgaht& zBoop&vV8(tPlb@)WtI+BhB)k~UMn&YwO1P9>poD~FW+bsvoCbPzmti+Zi`S z6v#(Lh9@1Sn%o|%T$mrUV`Ge8*;qd=?ltL=KH1|I<$H|r!Sw3mQ&_p}pDu zh3vS~x-v}lANxJB-Sqsd#4ld+NVRr#vs=lR&1sc7m6J5bN)j<+NCnH$Sa9Pg#b?oZ z3Y`|Hk>%P>kP~ix$Q5l^EAD&x8%yfT_U=X~nj1c@=rdAmZ2E%~KaUvlO&o(Sq=foA zUiV4xg3Us-Hfs3r^a$Pi@OW5x_e$d^nB#|5;+zCO1{uHqnEbKprl5%5j8y7IogFuO z@zVl$OI}PNArZ5)@E)BuOMde052a_7q&-NrupS{<+!c7|9S=xb;mox6R z?P4yCJ#cQlKYKoGX*4EHaB`Q!YqlCCyvtN4d_8-fGE~WH%Qmtn zn}mm&a?VXw%>|*TzZmCho3D$p-H+)8k++12esz{xJ8zFUbm25C$^b@|&$kHD|| zV7c*3lp^KlhjpuR;Gu=)q{)E8I_FQiL%}( z%hZqWoHn{vlP|Z`SN$A6Em`D3+VU##%qR}x0pB`vx#Wu|6Bj!MOO_be%@fmIFL(*5 zSR{L=)>T<1`LpaW)$i#1t4+p5Lm3cb7OCk2{&%BO& zp4Sw8obKA45JS>Tc%8I}Qc7Nj%;fnRd+v(Kce zx0#aMgx%a=g}WEYoU)ImBc|O8i^P*EH(T7_xW$0z(oK>rlI742w z8aS$m@fg7wL&_6@1W6udFUMiSXUXlc@lj$k9!0r~)2jtkBS$QW{;HtyIG}C|&q$%z z_>>biGs}gkZZq?~N#Q61C0p{PC4VNBVK;jOa%xQh#z!TcXa$XTG#>Ax4YBX~9T7A= zVw!$7k6cV{IgPALongpk4H#<}@L1$kj};Mwr8e0|TL&=G4TcZ$27qtR&%=L*8+-H6 z23vEzvfeiB3wrY%7vwTtI{C#yOqD&@cNA7)szeYpW2w5e?arT0+=Js#`lRyvOk;Ka zsczMD0ZoBZv(<&1hZ+{$dpa5F07V((%wiQ_X$4ta@(yKo-;aW<@|g=}icexCw_)E9 z;PAA#G80pdM6SA7T1JMhW)P~A9eX7{67rx!yki(wJ? zmA3t2GI}Y35ur%50iIxkA+pIIZp^ySVLJb>H`jT)&lhmQtSS}yLXprii=TrB^&BKE z9GT}tA!!?y-Z;0<_{oQ*PC(o(Ni2BUzJr%m{z`M5`oV_^`a5UtKjr-QIui8b-*Pd+9Cc7iox%#hJpz?o77+mMh*IcZIKW+(g98>))};S6&uS=PL`p z&88~HR=dH3hPCwpXn@7-V?-1m=#2?H+{M=IH=LinwL}`L`+pccwZUe|%j`LjS1aNs z7sM3kCi>pbitxha~N|<)GWd~0PO8@7~v-k6VLocT;X>5 zwsY|%?=PmI;D(8t;y&#uZJKsbY65yDvrop%4A}U2hGn+B-ThHg`tQrpj(vslrSn(! z*RUN*i2r!}DRpdwha%DAw>_~SRGdhO*8xZawgb3pBE;2s)uUS;f?6LCw3!6DBYL`i zjQ!Xa;$a%Xe-=~|=(QKYC)|pC_2&x6(U?eNB4jeM3dVO-^$eqbZR0D&W?I)91Rn{3 z_Z4VNM-^-)n0zuWJ^W-JfCG-?7mH4HrSx}*WfTJM$k}(TVR~bn8f>(DRk-xS(GTlcWEHb!J{C@u#zM358Mr?wVRZWQHgk2)fFF-13`?>Um`-gV&71F{*cgk zB++$@X^T8J>ih$C9Q54QloRRv69SXclP;)(8bNP)WJ^{Lh=N;=F`(d0>g!hL z5v|u^x2^D@ZImNop(0jQ&($^6t8Ppc^yMh-&+Z~hHKu5v3ZlQ}dkwat)6i`{yg5cB zx;*;u$9D%PxVv9r=6dlau}S`DWi5wxR0MbKCEGLcdN2WbtDoa%2zc#R_Sb9C`REQ* z4weVsuV0HFxO{#Eo<-h}EE^l>9&@o#%;Kw z^0Nl1PVvknB;l+_KC_QIcUGLms}=FH{{I6FmOeO!I${@&e@}(|gKVk6m+JwSr}#7P za{-puh*4Zr5<#BztCeAn<&HNzH(*_3Bn~8CVY_Yk9vQrsx%{b2)>mwmr7ISBN(Sp4 zPRvXB*vwF^nuE}8$+unKBAf*;a6c*;KaJhc^e9avOpcuy)v*?-yBt3kUz^``I-C$w z>00ceDmyBJEljChxpb}<4VV<88=)KJ7TV_Tvg_s_XrIyp)s>cyexhdECT_XES`H1= zIR@?xqM0!HBvQgjaq^yo=41^G(HCiLb5Nk;#~2!m<-(hb_5czR77X0KRLrsA4m;Le zRj(d`5(VYI?EHOcVK~s8RilmJ@nAZS!CdjACT6(uIYBdqD#lm#XhV3YE4@(qTb{bC zTG5PQIM7?uJ4Nl{!HieU_!8MSt@W7)1GNnXQpGtFG~}{6Sr5U} z4Ls(j_BUdEyz*I|)0*8Q@vFPkxkM?uS%HcRw)LZDpE z7&G3_@rsZJSfnYOskK@I>?xv>-X{WI363XtX>nh1u4FPIN`ki$Ph6`cJ73rGF zmeY=E4XKMf!VKh4=n5CGK9IRU5RO0>c~>#qQ)lxJ6<{xBMg@N-8{sAenOd880HzJLjA4RP+y`!hi35Vb_J9THg00S`u#oWgWfrIe-7Rz-qIvRE7d84$ z^scED<&DuBRbCcf|DU{BRHY83lD}tM5UxF>1FXE@%A1Dk+wrmM^OoX_=4tQ$sw{I> z2ECZj_RV^P6v4jM9j`QzVz_s9wZF8l(_%vlThur+ki^O>#o;|C4s2fJ3^BcKiK2{p z0b$ass8IbNY2$}zxcHF4Ys^ykcLn7sfp35XaO6o6J7228eLoLz?)pc4nP)Ymg};I| z%Cs(b>f3X*7Ne0|Fjkq)4vxIXzl<&qI)7?0T4Iv`mLLsmRu#S~=)S>mT8)V|U419KIEPc!UeNqTDlCm)gVEFyYQx zoi8oyYi}Du_*m1&3I55{#qoaiNIU;tXk~rXZdL5Bk2$ll6-pb&Oho(%)n$*E3f)r% zbcScZ(5QMB=seN#69pBogYf9cT^(NVy*v;b2WjGZ@pH5Nt{Nd_kTydQ%cJy!V@2&; zTN@M;RJHcSK+LEzF4P$8w-wDbGmqb)&du{Crc9+P&!a$PLqy=!9G*U1*`R9qCPs0* z@HAsYgJsfE($;-WP_RibdXefYJuHda_CC?I>P;#LHf4A`Du#-PSs+)$uC5fTR4d%G z{MUJ(R^G5s)uYt8ZMTIpt_^sD9Lk08(iLXW+DlVNVyZ~V)Ym^~;>pgCkw60Q*lQyZY`0In%G{ zVzcdmADHT8#)XdRw;OM)oR7N|8ka357)_pev{nAVl_D?s=*1_-js%Q-2!3MV8eLGN zspIE34tTJUlq@-2WitN7d%+RHoZihoPg-!=z%di_y+?&s&dIz`H%SIK;2QCk&gJyw zGB|tM6edxj!;ijR`s^mY2I$zEkGL^Zm2Z|jPFWzZ%H5M2zBPdln};Pj+yP|o@Mt6AaymERR(4iGuWwCX!`naKCUBLK(+>JVY7SihW)Cpx<4ezHONo#jbOFdU#PXsglP;;o~!+>$nN-TJ|Gi5(aKcM!>A<0Qj z=5=QHEla3r;dz4a{du_+sPWp6Oi(%FM21m(QOtgv7QsTHZtqJFe@x?kU9|g<$ZE-Y zyB$y*UhpdEK3v=zb z+w)F5QPU_3ZmE5>`&C}XbW9Qz`y#|CmthKtgZk6|t*nF)H{e)ddAR^A7InkB?6FHGxu8w(Z&FVKAzw&t_GkzArX7cgSTi zgdjlxmhuE7fwwz1IJ6!~u*HKg$Nx|p-14+s{hF*6Lz4uc!@*@Mmoei=62WtYxlYtm8^=t%OBN?#P=$7kM4TZF^17I}N%(48-w zWxbU+e-TGtoNy(Hxw*T@{3mn&HZ5tpuf3UcGwIVkLYAP5Q_Lg(IHPRhskN}k%I;sY zGt%*`9gv}!xfqU5x*hD-zbtK-n72pq)yQ}yy7ZY17fyA^ z$wATuFX86K0}kHmn(%3m?r^&^T*XFJ)0PWcH1Wj8sk}8QBwpK{o#4x`D>{2AX7~bl zuHLch`9F*vU6a?okwu4VvaCI3WBV$+27f(#2`QT*2}HQnj6Lh7H=F&M&5w%1BD44& zy{vrM`Yb21yGXsP8+PHjj2U_126ba(=0j$V%QTOuey3t&DbEDtCO#`TW#IDUIJj{$^op z3HzG?_Jhd-Z7au^bH2kgotqW^%LAd9`r@Uk9Rl1x34bzDT+^S8p=ztUn!nFZC9M=! zkl`g4Y7#U%c2JPl`xu#T{RMWzv zE}sz^ktbMo$a1j>z!tpdmB%HfLiQ=61V&_JWF+l*aEn0#ejLroSJAzE<_c0--TJ)= z>!WKqbk%nnZ*?z*LNkVjcinnd8f$9_(FhnZxBkv&W>WUI2u+4QFS-8eOZo8IBBx}l zbG)^Q*Z?EAkrFiqFHN;Z22y2XI;dCDEZoA*^lo z(0O_)RfPbBwEKDrlwBO-pMGyb~@S*=ox$JkO@gDA}Fg?yekiq}?7W z;(>#((2+*6pNajw1I+>bVqVx_9_rF4JB0)TvO_bm+=B;Z)$$3l=dGtz)wCd1@P4bMcG;rO#4yl zoh6fzv0w*j!No&>GT5&=^|*kl{l1$WakIo1G7Rv|8Q|La`|tOiy!Xgq@VZPYbPm0- zlHauD{6w&C4-t6Du^^Lu2e;v8#vimqS60i$q72h?Zu>twd-BA3$K~MTs8Fl$3h);c z79s+6KUL7i*;9Yd+Np1>E5haO+Qa+Wn~s6bLUmu7m(n@?zP|?ONex=O@rH+oAxzqB z@aw@)llatBQu!hSxyWmkB6T3G>$abJ26_AX!lu~*SX4%)E0s(BbDK`cLKR0Yb65Rr zbT17bfUF7^c(4eZl!F1(FtbRDSNhL6Hr|(I{c`t|n5=vogR#84+Q|+GF>mWZX>XVD z7$_J_8?G&zL})A11W(wu+@ON0qJ8I|JbQXaEISyiG=HG5K{A`_^*{-(07+W9*dpM|L&iGT91?Dsf!wtwk)IPeAp)89m>?9Qjo2K9I zqFB=T%reCD61}OZ;ekz7y9Isygi@LvUl?ttzr zrZBH}TpSqv5H16nn}D?YCoCEBRVHTcI%ZSk-1+XFIhzr)P&G2|?+CCokTO^%3P;zL z>W`K?B0;sbdStnCY4LVce;cRZ3=p=H&oEUZ&;U3Iw<1vWfZTq$#SuD99*2B$u@_R6 zW1NG~(=?}-Iq>6+o66WUgG&MDI3kzj-zxKy>cXxwp)!=~u-rxj0?ltUjZ?W7k zCAat809oX4@(q|Vh-O2+oK#;W?wc*Dq;91SWBjsZjP}FS>XIg`SGTsqKn;gSK^*}D z1_$$~XFYp7`(^*8#!KH@6#~LaE+dzl^Lzwdr7^DqGuAqH0hzH3Y(6*ZT+uJ8h zx9YYB-5po0QV%&z@w#q#_!@lEiE}cZp9a<0_A9bP`lAJx zawP9)=W{mTf}*AAthv4zxsH!d(ChrusFR$W9Pl7gd8&{^175wME_z(l^XNCJ#qEbl zSCKEs`fRQNhAYEstrRlx>(d|z$dtB4bH#3PlD0ao%%EDLW1!X9-N(g?)A0m%kcV@+ z7~BfBTu)TIO5no61XI-@7MC0l9(%+%Jmg{(rmvARjAU3(ouBTgm|sjS zE_RyR-SID47cnJ>U64+T72!qw2hz4?a0Qv(ZHo@TMkL3o-x(p5fm*fStzv~_H3lhT zgG;`S0H7Y%JHCPoop4m=@8rWEN(c;Og;2Q692(zZ0*^&4c?V_FpEb#lj*1 z))?LrL|!*#b~A9sf=5ao*dv3zlOdelw8@$Ah^;Y^@$jCHAV`ZE9$_G^u34^ z7SA4NToO+o@+X`M7$uM}$FCmFdzPJ@FC^`6=Ifssnos`JHocnrV@dJcgHJO<&Z9`p zZe@3>wUSKH(quccZ02g}E7xohC`?!4L#zcNfBDFl$Sl*vwmv9xaGjVRXf!t z15&XukC;54SeQ*9mlkIlX00A<<2;xI(Ud(*zZZ<*84idwKUjKc&bdpy%kiRB=8g~d zf3hmKb|U}m5~*F@n05IpV$CC^K0qR!Kb0Q_#5$-S<&852Xgm;6A5p|oMKmBl(^7Q= zNRwBMZKUSr6Cfr2fAiq^)hBF|Hv+vEq7m_5FVtasY`z;-z(2rFCyfH zV+Pt=SNB)vJ_pdmPj8)$8jgF|9hCJhO)Slmv(@QWjLn@q-p10+RYFzgc=S}}zv&$& zHkHtfl8WKYU!y-beo6x)gSwj1zGu^p8VmS8{3S2`8gSXbqN$x^{2m1oblvCtSj>Td zMSV-+gDaoO@YkAp&OAYtV7aQ~#FNDEB_g`T^)Fn}9HGzrIb;IoqYeRDtQr zKN25Cei~X>Mj`{R?#^g-KoS(k=w5J`Po?n|x0FW)vIw+&xFPBgxwei}|$z=xM0(Px#nG3{2KgDJCcW_10ju^|hHyL^o=4gB2k)(OV(No|<-V^$ng5 zZvdBBmsvHlNA--c(L~r{oJ8xC4;GuYer}?G#AM;ydP9t`g?icA8?%957O`;@^PgL5 z3lRyJl*zvaiYPQ!1^E^oq5~NKRjuE3*j49`Ni=n*C%2vcv?nQNzGQIu8LR*H?x?P= zzN=@LEbH3SK+q5GHAw{oiBsM_N_rmv2gx$+07x*Rya>GnoqL7ILPz_yJD$5#Z_dTr z3A=@$&N+$v{xHZ{n--!uS<$^zX$|Lk8BV5U+~R@_9sZy&UX3q#ze@8BX}VU};!z6( zUHVSLvEKSnLf`?uueOGti;1*tqu`3rf9WH4dgT{nTRY|Wxp!Nt zXf)f0m15lT<=cV~7U2o<7<&CE`#*ozB5V1cq&B&7kL*61uE6^*#{c+?+INZtGW6cfSlI@go`EwD9SCgjI7*4gsS|YV40x*7FoXT~iX9 z6r(CRWf!a$FO&1?@;u}Z7BwqsNpnA@q~PcVmuo&<-C1*}uK#X9u0-;hY-;l8@JwRp zlo>GrCp-k9>*;nge-U}fMfQKzH3_Wi`a|{u;a7-Bsd1hN8?J%IwvrH{p<0HM%`inE z$YG!<*F|5Jw=y`few+Up4Z?NHAkfp##1)%HDA(W4yFNO|i@IoB|0tFfsZSU3wTAdF zZ*7U#TF7_~ui0F{Gwa<_;%#}qZj`X)29q_a{Z6q}#6r>EF~15vqh=g97c6#-E~!?& zQTbJuF*T#Hmu|;?S23p7oJ^i2MJsA7GeIY36!JD}&$?`S?LjPvLt1lN;QuaP;7%eH zLj%4PXa)+|ap&$3ZpZR%EN?g+RnkVA$5wey9lwFGksj4Ce}M>YTGeyoEACR%_pWJ# zll>g?saNi8s2q;Ics5|dbwVWRDLM8d!C>RoCNk48Q_DS2j$&>9ozI5JZp_pX&D8VB z3e~InPs*KbFKm8Ox6G8Bz>Y9b+^iD)ehlM5<0vb(`FAtWf3-+>$q4`uRHu!M~@ zACseog8=;a#kcD)vogiw5Wa2E@iN5n8d=;M(Rr&hZ$fW5YRHQS3jSkfMendWI_$a= zccLqMdE1Hrg>MEqS!5ZN9C#$!cGFDq7cU=_bSw8r4+b%Uk~TWC%^iKd?M5JY_3Cc+ zv^@b-h@hueVfP&?c{RPNJ|SEGTozWG9*Zs2XDryPc1zm4hIUilzu#)CP{brYSA3QY zk%A);NO8;|U=c-v06Cuj$xPFrwBwVXWvB86qaGXYEyxD{F9=R#cq@?~BK*hH=cA6c?qMRGe#y1qOP}ht-7M%6d9~+e5 zL;lQEcp9DG?1}Mai}1*DjYinyfS96s5SQ65`4H_0&*%n)BJj(YHE+ML555h!RbwDP zWNtnwInBmXr2ypm)zmSoWdDH=lrjb2*5A>C1T{SE{^d9|P z_p0r+^k)Vxhrg+xNLa}+nT_|x7@}h#{1Z{|-jaERXxC4h59m<~3Idu$p&;ETi0Wp! zJ(zM@c;Vr<1E!jdY8V6J`h9!VmPcawtMclD$vcq50`UyqY%(4ZY`v&Gfz%1;2515; zGiR|+%NmN!9xCsk;J5Hy*0XQW)xdJOoc7~^G#iW>E}Gs^%hsnoH{U7-P*V1rY}8mx z+-5?2xPN(A9BUDhtEs?c(_3srCf5@>F1`;oqdJwc^yP^`<%kM4)j*Y35d6OgKf3Cd z`{MpT!Vh*hDH;0dNHmxocG=DaYQMH(&xosxRgRVgDcJ8KS125}uDT4(ZDV9_IsmjY z#y@(78TT9k0%VW0{ktn16(W+SJ%N<3{_Ki{h2i|%OfM~ntp6W;y#-j6ZMQAF07ayc zl$1~rq$EX3TBSp}rICv`_y zzGsX%#+*8@+CPo_UA6i~xzM^(;yalty3btru@76yk!FpSx(8d{ytVEB!6Bg|?~#=8 zadXnkAHKPp3HMI$rhaC1;cl_u@p!7sezIx65abkx9z(2+O-*~i0t49KMb{@W<1`5O zy59%2>*}pk@#OlMAo#qeQG3T;0zZ-L_Q zN=A!eExA|vu7AGP6UIVr_@te}jYp8BZZF7)J8aOvJ;4xcMvDNU3Y6qmHiE906?j<(GHa z^t+3VchNb+$jmS-0Baa$`c;T=Vk$tc^q2XK9babh+1E5 zkYUx%A_co~9v&s7H9_S^#R9HW;-)(^>Q;gjDbxf3I??o*Vslw5(Cdgm)oik4123n5z1J!{e|+EfAJ;i0!& z@Wh&K4r~7R@_A`UmUU;5p-@Us^4HJkK?e&6M7CTGLBcF6(j+M5ON=NIBZ%dw+N4J7 zC)ct>xQHkjaDx*`Ujcut8dr&!7-#sXeGywJ#7vzW+G{JzzJX01Cz;oDet@gLu@l1UJrHh3E>4$0Cbda#rf#p=RPs7*>HgS23EWv7OsRV(+^i4_mo^CJe*Kh7D zD}l`0&PBviDBya;-aVgs7L-a@d5;5VpCfUZfI}Y6)t(2Z>`1fAY#`w4#|R2-)t55^W4aN7p8=$t{YdM zGzjA){Ju{ed#!Qvtt2j(a?xuq?JlN-rkKaF@wPcUJc*SZ z;8iNiv^%f6%s5Z~z=v{aBTw)@vNNGH0Q?_}mJ<0OQgmr9C16Z^7%QUf%KN$`P@?pl zkay-9PuO zbdP5fRbnrr$>XsB#pkRR9|uMOHi!Ub8b=XFFl|CsJv83kzWEj%=Z~w5W=Xm2Ps>D0 z^kd)clAJBlWOVWVxGoee7xnGXaCVoh@Fy*~t#v~Ff3vF9m`e7w{E>ivB8yu92YU8{ z+$3gaYhbHDAvKevl4vL4;MJ?;6_!t z?ujR{O;`9N8O5n|6MM9u$x3^VEMpIk9$Vo=I!h%5N|UX>UTD{PLDQm303m~~hoaI$ z0Ld}1u*9@QQK7>^BWp)W^Rc8g9XCetnkEqyTk|Ric&l`J0s~up$W9A~>9%&^&r})?k+rmleJ2o2CY7d3 zt}0nCLl09JStAbZ*FDZo+3go(CFn)-%GindUWh>iwUJ`q#TWUj<)eIbVIW}sfVL$}4mzQZk|7sLqtk-3m^ zj|C=7`9TBtV{jtzvmM8pvzE&zP5!T0AD z@9|M3Q(D{kWaN;N+`Rixa!awl3>td~37=k9x}9#vbir=Gi(=8-lWJye9oXJSXup6f zC#na)z4HD0o(bt}=}^l;g8E8*z4+7w9E=Bb zlBeN9PPuaRLn2nQ?4Fcuu4zHOs>@foLq5JitT!=%R}0t@6=OZU7}5F%kLBi1b5QOd z;;tm>hHR3ZTz@3hul1h*L7JY27Gg!N-hC};oB}0ixMh4dJhl1v6Q<~kxw&|6G_r}W z_F^~@C6#jVish9)C-c^OMk#2qli9Z`v4rr6eZ&#si>&P!AM&K4U;F)!X-JUhi=>djrRN=Eia)1hf4Dky0uWL ztnbu;_mOBF881D4+mrg~^|`R3DV5oAq5JoQnZ;xtixL-*-U15E`g$o?5RQgNLZ8kE|fBYrD5#!)3!S5hu|XTGe^z(QHkr@|-n_(9=bU)kRgwt&^aYxuOoj4-3XNBI2?Ex3nBGG@g$5x)v3zWo zrmCZ_Ak+_z?-nf`dl@J(&NF3d72CeKe>GLR;hLvEc^GS=s(#nc#rDxzlS`QNDzYaW zma^FUwaN&`A`I8N1-u+O9-NyMQ8p;=z-i0J1 zSBeVHXvQkTWu7ZAlBGnj11kD))RX+^3fumY$DdFxW%9*=ZOI*h%QwR2FZsm2^|4Oe z;CH+@#Z&WV?>qbkhtu{jRrHd$TWmlm<~Pi-gii6^6a}tURRvy4I|nU-S<}n3t|12g zW;1R4hXIc`JS7-j8VTARwW}7LsD88|S|^3={k>3E)CIHduE1rV#sDKbc*Fj8rTXJU z!YaD#{^>?@RltYq#HB*6SDnCH76FZ+=QyeX+5u-9MEdJ!g?uGtlVYbH^9YVrCqcYPralVx)G4 zQn!|W=IfjwcZmNdK})TCBGt9;^f-Y?W6PXpYwC zF(UD#ex*I`*Unry*aIlw@slSVqqk%dePp|AUbc@N^L7J<*Zrd8@><>+!Y5txX7SqS z!=<7(IfPYy0OsaX=SLm=WN+@-){29@#&)r2Y<2eY>9g~wJH={GpVGY;FGm^Ke6%nZ zqUKbMiWn|)^cF7QsVKQKz*l`B1Du%1c=8ZXHv9=a=$2zj2c3!so$KkR7w8M0dGSh0 zmhxD*VPBHv^}IJR-$o{vFm}6T{7Q5DC!;FeA#T~{s;_nM5SzGmq@$>Gg1OWix9nfW zXAsW2hAWhr(RNw->%|fYC9j*b+^>-C)+i9`O!#S*X+d~=86i8cu+m@Y%Ku$<;n^Tf z9lp@mwIqu%hs@eu|4r$Gc=hL22@+j zM!C?gB~f>^)e%o``qFJ6 zrw&Da=ZBr9sE5RqI~IilveT&U;2qU7=Ppk7PAoQmefl2t`?Bawu4PLIE!Nd^(&IVz z0&_A5Zv>6-=aTQ0P#tD3>!EvZG<-}8l z+!YHf)3W#r&GhGeGxL?0l;$Vb{@mjR z&oF#Gm7tpul&^f-=^(@mF)ZuD9~Js4)PG_gWo5=3M18E3KWMhQi*ExaYWCbrLh&gF zQiy2&T^!QmSgJSwYOR0hBZzQGKcHQ)+x32~8Gn8Jx0>5Z4d)1xf{kk8>7@^euh?;r zeyhf9O#sf60cQ1rx6G{E`K67K^(8!B|3@ zKX{>wD%;1gU#&k5t}iV4r(M|4`B21jc^*hozb&!gQY}l~{A_Sb!^&;i=is(Lj}(U3 zl>2O~hR*v;t&&ZwnEH;stIH-Vr}dw~O(h>$RF2rO_EK;bL|8LRsy=*U8&Q>XHKr?W zBWrW%V*ZA?B4g);!p-9A8G8qfZ8|TnQ=H=8`tYMCL;P(1h8;VWlDxMQdMM_YSFM@~ zm;NGWBGy9-1*@b1;X^XCQTvVghqHHl zO?~v06y#fp!$l9MLXi?~t1C{F;id>DhorwH!5!PCdXog0P|FsUSDvw`EMNbV>GeoM z3DLsd1ry;lqm=9JNmAo+lN9{GvPm90zdbzb&-vuHFX23Th21x^gRA>+!88yB8=P&s zCZS(atS?RU^=pd(?g{Y`ep+k~Y1ItClSCbTn-*sKxV%+0yesl??6#DQg-=tL{72H4 zs}nKF)`KaGMY#4f!4g((2%)3rJP@C=QssOv&k8io2ggq5<-`s?>PA@er*F5K7lin} zJ;CSasRJvQX$njeFWtM@Y=hI?@lsHRu5kTk(0Qx@2)q%==cjtP1d?S=rH*t}f|&0f z8uRGH2M5xiXX+5?a`|f}2qmbWyectCw9K_`1ooHbx7^<#;|0*Hv3+34gmyXXCgQj+0%p?Dn!@Bk9TyIqr?u+`Mj0EobWx zJ>9#di3p4?9LyJ7nIw}rdQ@&pg-2sr6i2DUFnqY64c?429}96%6NT!SLjo62@7bUC ziHx@iFY?r$hkrDGglqe4aG<2LDEs=wNu2JLEQ>Vq6WxhOFd!W-`vJG&cwimogTOzCNVn7{#{>r@C}0 z$$Aa?mAV&n-J}~M14gDR+6$*optk!P4$q}asblN6{BB7SRQ8Ko;>MtEN#hHaymsrJ zPUe8cXST;u>JG!^Be};%OurqpBaM^#V+Q7439oddU2nFuI&nBOel!X zwKrHZ=Hp=^3@p^=mOm`V9Qcm%Q8#MKNuTGfoz>nT*lT6$CH+};_j`piwK!Btc@i_8 zW=CI$b^R>y-0Bj{{H9IsL>iJw%7;4OuatEOX6+8;nXG{%9>1~}#Tjt7&jpVOb=kB_ z@PXHKnTeSR(mq-zSNdKNnZCrcIvEDG_{Kk&B?G@|5qUO=m_mm2>KJ4HOe2({b}Lh& ziuqL3HS2o6qWIDB4HEvOmR}v>=6ddCHn!WHf`;SxxZJ&bf5lDw8HGj?Hu5?F0=F}< zVSQ1{FMMO@Y&OyT%W}vAzH9Hy{K){>G?oaEZIPwoypEV=Z~neuUtyuZIK4C%0;Ig; zJOETNzdg513}e7_MFadhAnWv=&2?mS6DlP10I|8bgD*6w+n1F5)UMAyY-3Ei`OlR< zi$oRkEIeO&SFZ4;Cnx-K-~*09s=Zz3k_s8$JONb7=V~=ivrNG4V`~i} zXr(7SaRJ*p@$6tb|8}KJ@GB+3Rm#SG(sz5rtu=4Mzxc3!ZpwJ2gFEugfF{Yrc$CNS ze4Pc_ibQlq@FW_wA@j)nc}rs|FLxz)YYejZ zHJ`iDAg@+M?`h;*;z#lG@2@=hg<+$7p7?}XaD(kH8RnTHV8-)CtG5NHma6JMf`JlQ zy%#LW4Z`5hd5^vW$?*z<^x1K>P4rg>ex>&^q?MNR5lRDLRWBOC475=Db)=j%&pv`s zFg+qM5X5+el@bm8fZGOXwtps@F#<3USt@x#&V{USW^w~fb(Ni*Bf)#G;yQ%P%%nM6 zYnCT}om+6Q+1q7Sdk^1M%cgsRL%%)R!5cC$dc;ZcCll z$Ssg1InB&@8#x~@8=mO(f2$}*hq5AU?)*-}7)~qm|D<*Qn_1Hb%4GyW1rMM$?8A~; zKu;hUHrVd{a;_)S=#G7}`ka54UvQey>8ALz`M-W0BNV4jD$;X&afMx+iXSO3ba_%8 zFa$z5B2{9#xl=_DAK8}fV#i!cvL;K4gBqFNv-a?&E#{qh!x*mLOgr6KzZ;O+a*}jmP+Y?CVw#%1{MY<)}2ZVCGKR^3DmUg>$*zvF6C`*0@UoeuA|8V`ngl;<(1`l>cY_b z03@nXmFZL{^r~p{qBNoJ-O{cn!b#9`N|VNv_0!0rERL2^lu8Jl8PHzjee}%eRw-yqDZvUlTWzu_s(q+?bb0)~ zJ9lU{W~FR&B*ttdbc5MZ{7$8Zk-hf!JJrgJoAaQHM7 zumc{j+@Xr(o-nC@qbmeg4=h6C2!xq2mPuH?)|Q6_GwAu>a%t$1K$S?tgn(t{jfcNG z*$TiA^=#S}g?NMOJsu@grnSgH9&B{4J=(pTR#v-bc87>D!d@F7(_&Jlo|NArr@1#~ z65xjwT$2e9_`=Msxe24MO~4cL37mDHna-V|h%79k7U=U9EgDzhAt3#pxjv046+@Bi z=N*QXvhbK-mH<>)X0i8+PR4z73+An^@~f;JE(MNBHg3G1N<7BR#w7;Lq1{S<6Q8Y# z@bX(nq%ae@XtXSUbH-|^xmc?d#g8Mqe--4jNMi<4PUJV7Ox3)SWu_*L=qk5W!Qf7P znF`uY-;ph}HkZUDyE}%$O5}loo<2+@;hOYLSH(uv<|LK=%9{uXG+44W&kN#GD(RWap z%~bQUVDijO+&>c@%(~Vg!cYvydprauTmmVG^j{9W?u=CJf@=|Vv8cc`cM-V+?M*O1 zXUYrKNWa4slb`>Tl`5k2gHAhPI6h zwkW@;40l`mDail80U2cWPriVCYd4js1gToxuOiG<*CXw+odFh2ff`3&c2O?(17rW2 z$ND{imtZ+(H>Cpj%H$)Erxebc#D_q=UYf%v{HOJ4$m>HMDXmm=h(-A>#E8cQKKU}0_eYp8_gkLV%W*5QBZ2gr_s@@Fm?v5Ufq9*LLTS0`bM3+i$HY7R^2zSO`r z^ohKI=H#qC4dC#{WVGafpUtHoUXNJnemId$-08SudoTXndW{+-ni_$fYB4c=36L`5 z$lI90i`NULRF{9))$hd@l!{H{HqD05)UPyNHd1`65j1+O;WbL(k}q=k=*L3YH+ zUR+me)x-iDY(-MxFBti~Ge=k6Wlm>WKToH9rlSly3Jnzf2n3y9vqn`!ttkSUM05`t z9O~%kvX!|Arb3GXk|Cw)4O|EZgc9dlaS7G^)i(D*PFgFl!&z$0^B6l|8##uh2OeG6y1nV{A zZ&cZGkXE*Q^L$b6%1`U~i%&+E+{CH+zv{3xHm8rp{aQNuh<T4U3KlPBbNZ_W}C) zM+%){7&%15vOA%gG<7^IO1Un(_gMSRx0m~a`_u(msvegg=pNw>T75HB56qlFAs`v6 zME__JPD@6$VtRC7VYvoc{=21>mE{Lnlk>mZCM4_9n=(=#4fqcUUr<2b1;gb-^E2{a zkN1O6m`1q&cMb4gi;=;M9b`io$?Oy?V!o;&_O;0l_plL3C0@e!ztJ2mGc@AAmR|H} z?L?|jEN?l~hfv*AKgVid8|djfEnMHzsJFAGu1@q?`fY__#l384vZm9}mw#H1?Yk=2 zB@56eq&dACp`yn+iuJpv07*>~&j%iAJOn$Se-26Lf$tU+Homw0#hk%~V)`zi+G>BaV+7x&)RN`GT|cjS#W5_5UiC_7Sw97Axz>ueA)HMa_c7nK zvx}674&*LhNJwI+7GYZ7JGqMSMDu^n95dsw%a6B7of}xLz zalKW7;WXBdJjX_Qp{*i64(m`FFrb?bOH-M5=QOipy0nQU3wZQO1eosf4JscFo950e z{yyG*N-F{-F~%RcQilvQSW=a0Q4qn)ZXJ za;ibJYFn?#YfQ>uCvuHx#ZN6)S*TDKVpX-oNFOy>)zWuA2Wy1t^`eai*RAg;zwSG{vwh#n!_qpD=k?cTfa!cZvHO6sf$#kG*-!(U z4Ca)=j(N)STk#J7(7*>v2);#JHu~CJ`tjWFQo79elJdE%Hl#lyUZhX}EB6hrkq*+! zm@dX7>p{bHv_Qqr8T;?&z{>zCgodX_I!|>^puqpz_j*ApO6%`$ON3%cUNAG9(hK)w z&fc)|X_va?pA-6|dJ4mpJ&Ruj+S)|%jjOvgIXGk%w0}u(aFE7^vqV>Liu$yxx^m1h ztnchi%Il6!H0#_{i_dHQnmA8S=pn>L)b$UW8wx=nM3(eg$4ohUtLtR)nr@{_`7mY5 zAC3w^wDGkNv9itn9*vi<4pB!RA|HoCpmR79SjPrNCelX7r%z8UDP20mT~GSqd9(%P zd1m+O;j}s%!l%6}Z7eRU`2>I+1-lvo1Hvkfu4hJR5)`)(h)}sfM=j=JwgZIEk?<=q zE4LfC*46%Y%4bL?yVYh>R7o{8u&pN}y8W7c-ixgTYGJf@d#>2iU=Z*m$DP!5x*1@ z1t6m50&Cw-DNB5&2?U7c{+C=_T|>@X0{tsvD4J!%i3a%{UpTIEi2VA)dQYjbgTkF< zWZb>y%4KHBPb~suUqdj04C>&1MH$W9JfuIWmF!3!X5^g!@y};e7_k7Z}U8n9_i4BPf)A+ei<$tMzMHe&k2MN6vTI0+CPdR z`bXjLMbDfjBUpk)P@w!hs?j)1Pb`vhd!T@DP%G^PM-d8n(OzmqaoIS^_et|3Nh_ZS zR=5&M)B*94y7th+w6Pbwh;9GBG1@Z_dV>(lk^)wQB#~ zxXuItDM=&*1!iW7NzeuadeiU>d>}JF!qYEI4iwhC>~sXIwMbl{VvpP5zW?ecNgWi7lEQ1Y{4Hh?_p6@#~ zc9P@WR>Q4Dwx(Og$;zOBjv{7`giN9^RWe6%HbJ;?0p_(D`r`6+#KXfLyZgX89`R6IWs%M+2o&)x3Jx7@q)z-D-B8 zWiRjz{Q)KI$)AaC@XZxje>o!bi+VWNib1~rp2yhv!2_VxKsz<$Rn^}?wp@g+#o_&Z zI7%Owi}`WJJ<4&jhyOcsp&GjR!H28-nLY~NJOKcsVc*UHUF9}~H3K&`fcNZ1el)NB z78@oSR99cXhf5O6%^;r<9o?({ycp0a``@b-^Uq{HST{w$WQ(8k1C^AJ;7sk78^5o zgly%5CfkS%LLryuIhl?bY>!EWtX1x8cB~gz~EpIj6hIXnZ@qVaVKdb!?iH(C3i0eke(>I+H1NQ4wpE5dg{)w?Dv4Xx=|8~ut)2D}5PUceQX!e#r{wuT_#;}%q@0_Wh4KK%2FV1byI|2LwySx3j zgPX_c@sL6(ui^is)xkUc1^-4-17ujGoDJxL&z|Kb@HPY-H_X`cNjbdJTCeX8uepQK_qjFp3~Q3GI8x_@0i;{{p6Rhi}5Y-^hIW z6pIGB#UOhNc6CXanB0nsi;H^u*1x=*52)$i2TD%x0%5+|U4;I|WVNlM zqlB{Zn{J+uX#JDNyM6L@=clekrJp}DfGQRh(c9ZQy|%{U^x@AvUNXEbp?@Aqq;1Y> z@|QTTe=5KhwnqsvZL_!bjx}3?M;4aV(7XY`=_%iU+5mgkiz^$oKRx{xbbd$2!ND-Hb-}>Z`p;%A)qJ$%hTj}FcFlQ$yi=dftjl4ygKZ=x@uK*vR*m- z#lUaC($(F4u8CBSD2)cGOMlO*!_NqlHd7;oLdUEtJ95`n3Tm74bt)B`GI9+Wld?9;L%&VdAYk++G^`b zg#-o!I6iQOfo#%`9(|{e6nF{u%f-!Y$1e<~uZK~=BJ|HpF<*6x)8wBJH> zsu=@w?>WlUtz!5br*@^0rKrl!S|=iu1w(o*dzM3$B?l@-HWnMt!{3kQ4VWE!ynDg& zBO)7txP9=;=zOj3WLM}~w5RL*zSL$Ao)=ynJLcZWOHYg)T&Z9+43CMkM%GBT@&uk>lsV!)ehW&;fIOv;^kA;RJ_-EZIFQ|OePD{HC%(SPGIr;g` zFclDi$dLPV?_}J0%JX(^q{VVpM#3_Wg~PE!1D>$^!McQ`B)YEGk@1XuR#w(|!fMkk!y=Y8_R#G>8({VK@8ZLY4aFhb%Aay5X;YBlmT zZOqmCbdDza)2DD~tbv^1H(X*#`n9MC`4yPD2IpcHj1;;)hP;uX;pS!o9@Zk9*12dY zZ0`^16%Kb`IvW~v1VFojs}F;E%q%R(qi$Z=-rn|)lx^tug{C(tc^ctQpJ zkEQgy&$tl{X3Bj&P7@W5seH=dt|R+FA!n!hrJ+tiIiP zLwiGtI@*4L*@5SJVZrDDU-O~bNU{7XYU_J>z{(W(ZOyGV#n@W9%LsPM$#DI>!CwYZ z+V)~9_3GeRvDPf9sa{3uU6Ajn$w;|$hwnSqQomv=rLNoNE}B`lGb-($W6N!-J$GiK zKE8N5ZWe(=wgSwLVer9u0xqR+Cjd#?@cTfH#X?2#kUC(QW z+&#OxmRmD#;&J$SmVsFgr-#6OLEY6dU>1CXWu0|iIj3+E&h*tpzjEe_v0;yR~f?D7!f5rbUc2{th$W>f>NZn0gS5 z{tCKgi#738Pr$snvEQ zT!&jpNI%$Dt5R_vMfi*a&wY)mv;8q>@^PlgHYlx;vwFZCZ1($(<-30Y90agmPQdN* z`UA{*y*R18poppwgtipJ zRj!=S7>1u6g-I&$TDbI&3KM_4Q{5N;ZlECoRt@2g^YMH;rWR?61R*~dzkdoV^Bs(# zQ7P+E2bomG@!_F5W%Yfc=CJF2Fw@j%d%kV>t1;H3=Yfua#f3FbccO!wayDxzt1b{r*zau9u((PHS7+ z(tu!ZUmpqro(t?I&9W}Amt(~VSu8RI_G_cuDquZfD@{aSSIL~`N<0mH@ zwO90q6+&n3^!!dAT53Y}HT(0C-Ql4KZU%-+mV>z{2!WH;BIHcXb{;ni3yb=UL@y@F zB8Md;R;7)gg&%XI;)`mlN2F?18H`P1zqdOs_O{} z2`(pVWrap9xH-y4%F4=H^;m4X$Hzv^Vc(yyRU_9zyu60v2DkO{?(ft{x?8s-`C1(O zn6+dQ1g<}cVQs4MJo#eYcMI$sGX6ZA_QU&P*o0{XZw=>6y~>3HViw!w{ySV;T>4AB zY1QN`$E!tMvm>U7CC5j9U76wZBqcHuM7~lCEc!2HB0oJ4hvJL=cU0u8H%1l~ZVDWW z@l*4j6bUlUbel+H~1$_E@!q5>gzjn5C%zCaL*)9Fgi=10h z*2jW~kkF4*&qL8=;%`Jo-F4q_`R&L`R8$mH7ZTmKnn<^1U$M=OmRJe~7B*r>n9;|w4TUqwnZ$OofG@1Kl2))x4myX^wF_9 zN@IpBy`L>zuK^x6{E;g+)~l8+niMjt@O$Zc_NV9L&i@XxI6e^(gGa7~xl~U;sF!xQ zv;gR+pveNMZoJoX!ssJ!!o20DM6qEQ4CFGPPU^|FCf5BQp9s4Y?$v+DU0A@sA{)K% zC*ATp_$5g%;#eGcN#)z)8eKU-@*MivE_wvQg2ee*_wo{X$)&EE;H;1d$~ zX_>NItNf@tRBgMEM@DmB*Q_8rdTvEbkN!UG^(AGK={5ODR79?h9nCYP55vY|mxYue zTfz14^t3yea5jP{Q&daW=~0R#F1 z{aNIxHOXTp$w4*>_x%k?_)C@Xq{&*wPS*`)UM%bUz^3M+7e!I;;xFetWp-+F^6LU( zRp2YvodtuwblJ_#P0(bg2yJa{)}_nFv)fDx0Ln!^czBi&NM0+9M;U=cVu>?pz;|lH z{L9wK&NPgSa=O{gTeo0j&$ggxRZw-0>%sJ3KgD^pcE53(FNrEvwnYv;IT5DmZ7<{} zk+XdK^eOuF>z8le5+R|wl!uKQ2jcX2XewWykmsSl8+d27p!Xld@HUbMokADArsFvVK@atuPv5y6jikb=BnLO`j)!;$NiU&nH9=% z2;e%srw~PTgDznu$L_~fTlP1Jn=o)NC2U2c%XU!2)N=I2YGkYV0vB29`Km(O2^5sVFcc4B0wfqHeMb}V-- zqKSE{@3L@!INkS=Q%)DtmIw8VUOTrRZaOuQ*Vtm#+ z6&=GB_RN|pEEDT?Eh!J$N5dA?HF4|sWQFhbd$crFr9k)i;t=J^y}XL^P+M8$i;|@> zlX!)_&k_SK)h0c!tCpxv2$`>G$l@*z8TrNmxbLWbUwuskaQ*D!YA^;Ydp_WwEdEm_K8_R zy~Pu~R|f^x^s-;KJfkz$Fh?S#v^j?rS~=JL^TOFjnNv+o|^>fQ3sl zoSpxx<&A@eEgA&kI|}VYCud+q^Jv)l=|ZA{Dt+yerz1ZV&&A9mm4T-ZZ@n{sXLVR0 zTTA?&0Qp2)1>CP8ZbVCeF>X*6m2K`RhBQhExz0D<{Mg)Fb|e|&g-fXwI=hFwR#+yD zZyJ2@Hm*T!c@N4`l0J+4j@kNfdX_r0CG34-Gvn5LuoE;VJgqJ#S+=5mw>nt3_UQJE zi%~7?h{KL2@()u7b`L%FPrpcEP?2_MG6>LoPzikb8uhem>YRfAQ)l|(>5ob_teGXM zS89f^ncz4ywYUp6GjG(~?T6#DiTo5=0dB9gK=c4iPtT1$XO_vuNSo8-v7$SSck0{| zYhJV!)7xob!&*b>A8k+*nlW|Dfs8^9Zvs!4P{de%EDnsuDR8>_YZA{uC)H>s6V(qMSn*~ zgKxh;z81TVy=g+lVB{uZB4dd{><4UwyK-f?aOG^?`=4z$;JD7x(40|$G35K?jh_77 zKu{?0uG*y9AdTSz&m6#Z=h2VEq7tK-qS=n*6L}*Jc zfTDjku@az|#Q#aoOmC-;yWYOqsD~<+$ByZIvE<@AZ%{3V`(*mYnB~P;g@U|Hqs!?+ zT82N+z4e6H1n}lgTuY~Oo1#Ca)9dtQwAZ~Jc@m%)7hoLIK1riYp4WVLlZ#6Bw6@lh znC> zpS*d0;9M|3*3%d&UmqwMD4G<6(;wZHW!`vI=)16D!2jUBQL~De$bcheDM#)ac_4Yl zl6>$7{lNtHElPy`{_coq{8DR|Yt~N1#F)!!@|!C*nkTRrDEb(Dl@T7?ffjIR4g8qv z@M~m)Klp%wr0f*%m#>{f`U{c{&1WQyj~K31?N1dLfg^LkdSpzUfIxm*(K#9Xfd11+ zbDG7pPEL2PW`ghOU85ZXvWJJIm&-n|5Kb@71;L3#)U|Qm&J&_A%4B)-qk^RgjuShZ zuASm!)Fam)@okN0E)_kKLZTy3%#(Gjk-W zKV^s7+B>I;8>%MDUT=^un@| zMd!1owT*pjXOgj#tZ#nUZalZVJK0cfrzrMD6yu7G$q)ZKPlZWIiJkj)WD{#Sa=DGM z@!V6a)v!E^cCA#_o+EBp2@Mj?aWo$^s|=M`gturOSD<3gyny{O+W&k)XvpzQd(Iv# z;Y1BOnkO1UWVk!@pQEvYG3l$1heTemlSviF-TRbMFks;>px2Wr@+v!RjLY+r-0iUJ z;?j}mvi7lbo+mOx>#G%oJso@x?$7N9lwqMPA6E`(X&X+TrfI*Tu$l;~jISK%+T%f# z5$RGiG$g!j`-vbEHR+RWuBgzM0t;yy#?IhSw<>QPw7%A9DkS26 z>drAmQk0OdL5OUZ)0!8Wtu4cOxOqoBydFt#_^hY0hW9Oy?!AcF#g(m;ZTWzn@SdmT zo^eIzEq#p3wE~A}n{F$ed52R|H5N`CB&x5zw5bj`8iMN{J`K8kvRNDy!QsK1FEOf? zuDtr((+2TVPpvMTAhk3s1?HY3Un#yV+zKIXOD|Mi%Xs>d+DCsYrW$2Lpdelo-ij|t zJhuxvZX6cRStM^zQ*Jgts_sd*CLA;^*%j`?$43Z_@A-aN&$5#`fL(3i6lNDLhj2Ol ze!D$E%yr-witB#qm*VE^yeC)55;Mv&oFlsyO~O(iu~hTIvCoy9JI;MylkY>JeER&l z#FM|vAdVIZwMOD6E8>%3k~xLL*3WnKw|=EqKW~Oh7(r3tyS1+N=K63BnJN*h~ZehiE9#ZFnlERsUu%=Yxk z>X88+mnIJUCp2AS56yvnww{`CBHSFi?RE7LS zpBpopV)1?OlNJg>i+Th#L>v{u$J|Lhx7H+kk+}0_Z*`Rzug%p&Am59vF+cJ zes+CGBu#N;>q=rBghGTulP4U=U&sIi+@I~x*8%W%*Un@9g`B0%1iKgBrL-YCLuzB=oDXNvd|?>mcs1l=y2wo_+Sp4G1eJp zb%YF4!+RY0tCsOKMQ6_qmTQ)dKR()Ot6$%wSwBWZP9Bhn`ZYi-9}1S(>b&8N$1WASQ|!}+I6=M2n`Ih%^0>rw{5d_ufg_Cmx+COw1ic$ zCSy}o0MR*OrEfP}c|05?spNP=-66KZ1{BF8IH?k^TayK|6| zK4GKdzE2!d7$O}iU$eAltn{Hf=#8n|tl^S`47kcWgy}a}nexYO$@98*+9}%c#o70- zt}xuZU!oNiPm~x2QQ_AS_t*HJfaVcPU7PQAthtn(aq$oF3T~~?%GX0Sy`VGO<@cU` zIec^2>a1@Cfk38+@jTa3l2gr8rg_-}2c8$R19E{!u5Zl(EmJJFT z)?*6<;xgeQ@rNo1NJj$!fC|xpfyV>{9{@7@E=A*c34t6^5C7K4+D4b}`&itmp@sgO-Pazy#Q(E8;JEhNm%3$|S6a6(-~QnrbWUH8R|Po9 zZT9F`;Nj^mN)LYn*CK#Hb5rmSXnPV^@T>vfWek-{VFlgO3Vv4Lmh?wvIYcz_e|FlVJjdR^V!t^L5m{dCGK!=Pg=yMZG^$~7^uvAx%> z0r#GO%wHtGb=$T@yLLrYE1c7y>Z~(RxC=d7DJMQ z&~6?jrP(K+7S(*c8VERs^b{lD$jTR~>kg`cy@gWE*|)Mm>pu3H9LwYZ z-o_0)qwNv@JCI@(zJK4m7k%RSAhH|i66Sx~p|#>-V9W}IKG?wkjHX3)59)V=6axby z3b+^mY$EXZq>ZN)7R7b=^c|2;U&jDVEsL4WCoW2K1fDzm^Cxh1rWNo~n2t3C6L%>X zX)*y9pa730f*O<}vEb4M;34k|Gp`$Ys7krTEYT|{D}A&q5Gt<)O$(sNSK8Ec7?A?W zOM>tf9ANX|YZ@S?Ly{#p=}r`d4aI|Ej;z!^Y84`~s1bmF{(m^v!yjr96U4y4pjrYv z-7qDwB-JXpC>2OC7#SED>Kd5o8kmO|np+tdS(zAU8yHv_7)S@5*o&eeH$NpatrE9} TEdByUpaup{S3j3^P6$ zqVy6#AfdA(HH1)Z^!(0w-#hdDbMMTZx!*UhGmLvbd-b)RwbxqDPK=?xCIj6KIuHoN zp!Gn_7z8?l1%b|L&{9)Y!j6*x%73SwmGzWCpsED=!>4B`_gwZ5jP*dE06q}tRRjoh zKv{aV3{``}?P-r|s3Ujc*Y93^sEH_P>0fB8?DxM5y8Af8BD|aXE0Pg0y}z2jkJL7Fm3$wm3yb&9 zD1VdNkodkVrUPHvwfqS=`mr5fTKhArY2u@AdPNFmPu)teR*@ zsIwEq`pd7vq~g&0)sNRb5&h*&ykP38qKp3RBE%n=caVxgqRV>um#j&V1J1ELGt1|Y5pIc5%Ta?`SldNF;v#{MkA1p+}<8i!=lO+6Mvk4}8irbe_mMCMdtG(R9yIvkV5S zn@KO>f&SfVaV4n+6k8Y>`l3cf3R=ByJsow9_bjboS!~n#rCM_;M_5p@TnX#n}AL8pg5v6J9DP(f9S9ZV8r>5M-#>TL)u(h?d z%F4?2_I5WnH&<6z7Z;b=+1W9Ls5DC8QrYQfs)3@X=7cTSmOj^T&8lVy!)g8n| zOwNFuMAn3LX8!)o)@T&8VUM&2tcNVl``;9QSFCb%Mx5uSM93`D#F<;-)S)!jXV1%D zOqY@PvrGSf^6vto5kER*_pQ7k@S9)l1#tVBSC4*oPl;E}*Bea6Uz?9=1pQKM2<-7I zy8$Q}D8iN5gu4!1`n<8IX-Ob~zWY(FBWuk-bU0fuhYNzUaj z@Kj*fq>b*UJa>?xK|5}qs@8zBY3aLek5_f<6q0?by1-f(g1~-M< z`&fEVr_%or{ol@}oj2yN*tuJh-+nGYUwZUE_%I!9dzw5FyeTIRy{GsmKEAW&CbDH_ zX3B}nnPD!@zkF1VEzoF`3!dqc_tMQ;uiW+G{zmuZi9YR#h`~>|AH)Or;dN9y5G#tY zjy5@j?&VAS*%5WZr6zV)___*?=w6jv*8_U1Op@!;dDc-r-PTPjxQg&O90bAYt}1>o zH=}yy#O4hG+r5DbUdg^@4HWTDjaf}rg5GBs;q*oDzu!ryWL+zXTVFIhz$t#1m5Ib^ zVt>DtzU^b3LU2mpl;!JpvKJsydCY=?VfbKL2+b1*q)xmMN-pqFZSS|trLoPSBK>P{%lKoBm;4IcHC8< zA#Dh>W(qt$yq+K=lbwjr*I8>vniLIdDsr34dI1rSSB8=66=O`C?;If?93Qll)3mQ- zNqbwdWo{6&v3`=l5aI&#+pLVwVgK$tKauOI_0h_q0?@uG0ZGW@idh~EKJ#VxEjLkh z{(J$ges(@D|Kh&+N%!UE&Uf}H^)g-&%8^y8f}H~^ zf_s1Vrev%R58ExGO`Eqq@Abx^+b47Tw2EJ{AEsA$3m2Jq<2<<;y(9Fu*Sea=%svz} z4~NUp5k5#~U)6M6I(pECbtY}h^AS~T4)sZLG+l(3V!=!Cq=XUeD6& zbC7P|BM4Zl^j+LDKrQW9;ij+m>oIA+_2FFmtyGC3Ca>^J1-|%fS!nc*gpg(aDw6YZ zRD2c=%gAt)Ei2W5?c<@J{ps`wG;KXNj3~W>Ri33YchX-o40Eni=qZ-X~BgD?Vw=V42%&cCpCrecB3Afs1Hy@S`>}TUzyJ zhRNm}Q=ia^0zk`O$j3J8^}OxW6BmtleGiYrToOpsuJS;_rJA3z!RCZ)s{e!+3CgQyOof58|i|amz9Wj zQ*5J^Fjh$W(&0^-j+JYgC!Yf5Ql$N@f}ao9#ZA9jy(_jo$PZQgnB5Z~_kL8WE5=-Q zh2W*h1qDUlEmX;ouNY~6^VCcTU)(8ORy!b=q%WhMSR9Dt+{`TVx!k7fAK6;l%Y9I% zQ2VrLn3mXMorRSW@wSSEtE;mWyw5g9UNP9OjU&p!Y`cT>dpDD0-mr9)c*@4CR`=wc zIa$$*l$T|dLdHsXQxA%Fwe(XTspNNQ`Ub`m3C5ucj1Za?rz=QA7A4{MloA53Rl4@u z4Z3UTnGHdRwCJP8hNI12m?esfyo#lkZzO0mngdZz&I$g>SZH_9XtIexYfe^x$>g-zDaUCIF0 z^xL~1CdHt2BSs^i4{-fWU>_?lyH{8P!@@3@%y=+w%@Skw%8@X%?IL3qo&WwLuBc0i zs2n8p_26AuP?s}Jx3!|;QX8;~9a!ZRUZ3Cz@{;|TJk{MInUIP#C>eq|Q(r|e^c-a+ zk>4HfcUK$&|57**xxMNf1lvfJb5>&75ar^6p0Ih(-y2{wqK>4KMM+4;z4)!AtLbBG z!p~)tBMYb)k>Jb@5Oa4(4?wUzW zO=-p==Y)ZCo;l>)%TY@^SU=MHLUoQC0#Jgl55rF~o?cAQr%AL3fy7h}o&1o302;TYw(^s8ZFGF1>;navoqC;XGjLT2h-8_3fqa=AK;W?EL$V|H|%7 zscZkILj0cs`TtFS@K###n}OWFo0S-Q9{H4Rtc#q--==5?o7C;#Sh|JwqO6{`l-7rWxYa%7M&Z%eS9N^#Z$BkK>GvcqCU&bCt%@+l z`$t}HJ6Q(5x#{9JA4REZ)gV*u&%_qCQEjfMvlZb-bRdj_c$+hd?kk>^dS&1w{@{q?JS7zihigbvwPkj5iFw}{uLONQe`D_SVK85KbMMzDZKuA7KZ{t{ z*2+kZYVHmF&pqlE6z9lz`d`G@5VQP99x-pOf3U{`jkASfntPT10k`*#O_Hndf?W8| z^M~C>HT7$A_x@RE!~)jeSN*2$dtF~Q2R%9)_rl6!6BpmyOZ9JtTNVZYqR#p<{A=); zSB^huyKs9mbsg{E6P*7b=z6Slv-_tX{pWd&0y1RO&zkM*KQbo&QkYGIPq6$0f9XQq zQ!bUiy#D9%e_@CFpjQh@Cdr9kiX9{0!$TmXutTVFbQn=ZRA;@-%)Wz!ZEQsBX*8o9wy3xDw34+u59a_h+I3#2Dw);&S5-Nk%ug%{~3qTNK-4S+>lh00Mp|^hN`9qskM*!R3RWVHb36;YxiVj#awWq%*w4kAW&Ona%Q&(Z)8O z4^!wP(WSZr&B6`ercCi~&iLmGh>@OO7ql<} zZ|MX61344T!)}q&pI|?xf>+%Hp)B<60dMEFn~#oEh+iD3AYLAk!~N+T z$e&v`*}0dcjS6f}DInn($aQy%y&kl2TAOZaqM2n{LQiD3;&o6}zF^PRthS^O^w5fn zY=I77`@HF3Z_&bGouQS54T2|ouNaLIgfO1+nRpxjP@4ibz0@tj)c{?)iXW{F4=(i& zZu7RayMzwPY|W~t4&FANFVa2>&#XL+7iHoXGR3}8d?MWU`{{@3Tg3G<`j>z5fQ(gh zf^y0Q%e_ZUjdnV}9H?c9?bRjx#OE-$(IPKvrJxStBln=5`NN!sB3orLZJgXt>Br^m zmLet(TTKN?$b*~b5NbA}scEDK??jkOP?d~lLMOya-_jkG^$1`%X?ApSWevLEVOS^s z;10hC&>bzalSwmok+RA2cOL;Qn*ZgV{`135kUx=Hi!f^UYFMHD9U|@z`B&f|BtfPqhBe%5^z&npY7Y zxAH$)4R3tL8Q2Q#=^2`^6Pn%HwrgvvYrh3AcGlsD(a-u-cnwUr4rQ2DmFpj{GH=O~ zmHkeT4wZ@NJ=#_86{q?20jzhW@Kv}EOOq{d>{6*Ry=|pm*?*Ft5L$&<<@7G?9b3aU_+l$!q6q#{tgfQ|Uby zEh6sum$#ZlxIt{S_S;K7uWz=fDM)#Ip25-VN^q4qftL$&DpytLO7l|d1D{>O=itS> zbNuo;tYx4f_m5l+ZlF|Ly&r1;N))zitb-Ti zTX-w?ZUi)bbZSaD;;0wpDzgz|Sw>e03nbfoSmM3=0I<349~|qSBc9i4We2g&k3%q1%(YJjE7bWZ|AZ07M zT-f@H#h4j`OivA*yd)8mi~_FpmeDixo`1j8zuZz#t=6{6ymc%O%}LM6E63PWev^|m zEK)HwZ{#%W_x{+bBGfO&lmv?pQ5YQm8738NIp+}U-|ZI-x})KjfYqTw3TfZ+&iQU= zT3$ZLs$urMeB>2Y2b*6lSTvd0_zN@bx)Z9hwZqp@>*$!oPfRaey=v55{Pu(JqY!eIe;I| zKe(^YsFjS0K2x8U@=)+Sw{wOvWDH4A>ThXZl39tg$Sz(w9=(7h@`2aiq5AjA7|lZq z=eW3Uc)Gc=LJ&!AKV5&PrgjXr!solM>co<&LY$p11}w0>_a?Tk@!m-8G1_Y>?&|H4 z323~h3$%1I9SgN$mr4u+lC8WHD-oTS6uv%-yiD^o_Wj!Bz^K4apQKjbo5)0G$%U9!Zz&_@2HeyGY59+vjfaph}=vPhU~! z<_~Kb!gPg7*Fz(P7i7Au7L*!+HgStmRrAFIWZR!t%a zPCN;LRJr$Qeau(?^drcfPkJE8QMp8ip=Nc09y2iqA?O8vFgabLj3m-^m*^aI)GqXc zowgn?e69I%%KuGx^E$>_?|b3%qM7b}_=9m!I z=!2~HvmU=E;%z3TK2htus6q%B$($ivdK&pElWyV(?zzjUJb0;G$;!mJV=G!;Qq?K& zh|obY=d+P8guh{=s6Qc<6CATd=UBc&Vna}+uhr3}&Xo(VXz?J`gE&882g8nkrOq20 z04^FUN!sU!UP@#WGKHL0400chUVig%@n<;AyAQY=0$!5^4n11@bh~NjN+B_;xL1U!{RI^mpVEGleTj0U_L@rdDgPe zlllVSIu?7PZ$IMuogrt44)?oK;z2$#lP&~ZgyW5doTBYE2yQ=BnE(45i`}+g+vk@x z_3w)Wv~AQIj8~{+2&~GL_7YGUe#B2vjzI_MP0>?f#iC`1N;9LvPcW_4rVkZY^6{-! z1tHJE7g47UN?J3`j=a=V>dJA&M)i)jqY@MNlOXumc z8afTFF^D9Ml-ZRCrwK&aKFxz~--0&Xk6cG?Gmi|PH|2A$E?y8oig6~M2m#s!OC&}i zpZXV(0}lthAKzUnZ~O*x$<^j(bKbRB!2}AiWCV4MA>MbV?YdiQTlA*UXP1Hpy!hUm zfby!A*^?g3*J={o#P+Xq$2mL{p82o>JZY#;V)D!uh^-XjLfhWt--5GbxbJrIq_A-!@NOfm|c?LQf(!AGy-y zs!VTv@?jST^cm@NU%Up9qQkVj_>3hxON*{8nfYc1mv7m9h(%x)V|jxje_Xd+Ue)Gv zY4cuydH3qi!&sT$1@|$SgTv|c^j~#!@D7O4?s$vb zB~n8(IzU2aWbWHukt8+Fl&H^ml zdmQ&cFF=mZASgiJIsAG0b+L^>lrQ#@!ob=#v?gn86Q@seYJ}$T=n-2G^;H94yNWIC z{WbKofHpwm@Auri!7duRw9;QUv96*7*xZPM0qSuxw2cokobO*D!TB~i58Q_5MUg`w z!Pz?hTw{x@rQ?9s>JMUB$Oqr`Zjp^|7kRSOehaS*g;~FZh?RUg)b-1a&XS|$9G&y%mKNQiGY=P!GI3j`XozD2cEq8Jx5?w|Z2u^$A>Qugh~V^8Ts+tIAI41= zwbRW7ElzGe4Rv*PY`N^U+wmRJRB|PtrWJS7_x|=JV&`tg)w%-N*|C#_)QlV%`;wW; z-UVyR*8{R5B|lH5Km8Evb0jzk$jdxYekrBB(w$wQ|AmFtRv-rQ(9fY7hS%3Z52{!o z+D{|pwBN9D_Y>bFULlxE3LL4{?SsP02_eeFE2exhKy0pu`ehhTBdWRJV_}V223E37TyH$ zdDPRn0nzs2;sfz?@Mr7DYhlF<)EBk+S^8_H-FK&N^Lx)3Gu%ngU~vxwwZ ze}%f6h+_S>{{%Xe0FPH1(&wTl?_9{eLXAXG{hOolc^q)|e~^?S{+}B{wK0OWfRc_G zC{FsCUOeqfygFre&8kv&F=;~U`cKepg-G_H_B|QOwlHM5G&v?#m5| zciL@%u?kyh#3@m$Hs-@&c*U%i6Vw!d1Kn1it!bVxP$=8oBD#h%+bAHk7?xTP^n>dy z1@&!dR8~!FdDC<0 zBnLnT2o6o)gN1>DFz_)C>FEg$xOQlYTmn#q`x(mX6lkTrpanZ(j)HufEl=YTf8roSBXN{u!~N__P!8bbK4NaYXaVS_$ZXu~P9w^Nyqyl3Wpc zJDJS6yMdY{)I&KzepJZtyXP{j<0cC9fRk<@h`8DUm#A-J{@0V0^yy4OP5@HpPT)B! zY6y*eHD>)jnjORviv9ZjL5(c{*1QU#`J%uDy~775OlOC^!cx6pLbK~K;22-Qlz=BL zl*2bH1<`~WN&Sl>QSSW>3O4Z-_Uo_`0BWScF&3WvgFnwryf%&S9#fDsq56Z_t_*mg zz)Pgs>^F>m5bq|!4uJuGUPq`@_#X~LdreUVj&%k_^{SO%|Tn_Kx(K4;g z&Og6rDe>HuOqmw}bIsV5Hbjc$7BNo*YkqqykHu$_7Wn0#EII5 zG(7xzyRjUa&}RzzD=_RdHlwF+hFV<7Fyzw~L?pGZ3OFBO@=mc|r)B0i0-9WQ9K z%&>WH*BxYR82q}E_z%LJ>w4UyE1r40YBkdr*$I(NPb5c~MuW_K+(S-Bg+?a# z@a|>0pFcv>YwGzT#x9OQ;BFkv3bu|yh~@A+d) zi_l{i_!eIOd(q6o?qgH|hrenSJL$(>(em7+HQT?$$~OY)345uhk1w)LB6h~{icOOR zru{)?y906`hi>^jJM9+-0~qdWCdqq}O-?rk5_qTFz06S&Sofg6PSDz)CA@>(5m8mx zX7K!7`_NS?Go@={7j%{KAF~Zl5^GtK>9Rxi;}tFiI?z0H0i(>7Q!w&qT9 zF1IW)pLET%tea%46lS?UdTNZ_JIpn`SAEL&_kRNH+y(BZWbYj>Mehl}tCZCCj~h>2 zx=mb5e=8y3^q(d1$?`89PjB3$GJk_i>6Vtrr$bsEx@l2CXf9U|ty_O?nRb(O8*>ie zf}$0)C>!V5|K}HJNWY_R{=XErMTNACQ@{b266J6?x458XRZ&>$ zQ;;ZZGGznb-%CX4C;^GEq<_;uvqKCh$M=6ZP$5+Tty7SO8!+@V?>~NeXwO-VC4~I? z8YAidE3~+Pnf~vCh+9OlpcTx)PxIEDALea)zEg|_ud$qmolpk|9-+P#0|H+Ga1lAvBc|uXP_C6G|Q_ zHidW|e)ZgK+xuL*ODoSM-&!Df-2j+92_B}aJ&of?!`5s&a`%MGw+g)B8NSJ1x;6XF z*bK-k=;)c*iu}w=j9(tkc$3Y}J-R!uQ8V!{ebT1di<>_rpUZ_h&iEsGA6zNo08B zep2vpDraC1$2UOfq{{1dKr;lZOW^wBzH9ZR}-(Y<0sKy#~2jcwpQ9(RR zZ+9?8uWfDSV)W-z0PF=j^{;cR6_^zxA7XHh;Gc+tI+WUmqAVC<%$A3D)gQ)jys| z;4qJ5NFh_3u2au`41eu&5KaUi7r*=!%v~&W+-S8d2HR(yhcU?uU|>6&G`MZeVOB)} zOvsL9Ldb&A#f4Tm97jBMNgY`L|Nyd+?GLE#clAhD}~s$ z@5VDi(9Q75AfSpsyLtby!C}YP^9d+Exl;aCFc*141#NR|558c_yA)zJaV&&@={g3q zs+Aoo8J--u&ZBJotbrsQxMp+m%fsljO=cpd zU%qWEVTGaOlpt(J>r2Hc+<=!r~z*~ouB#_+PJ-S0uy!3{$+kU4g+<_TXuz^c z84lxBlF6!ehVwWbx7~ptAZnh!c7|zr6MDwkrd1^~NKfQo8*$jNbgU8xmm3Y4up)l_dzhy(&kamiZeD@^mDgXzxv=FB-C8UJPcCABK^EZ_}LWOZcnU5(0fBM zk=Qe&W>*tBxa(3mS zES^cZc)GF~y2Am+mA0Rk0^CyZt)0_E+1-PJpHt0ddK_mQDqDW~lq|JYV*M9;Rfj8C zurw!G%u**uj9Rvv?3>>!FW45CwNX55yBWLRZT{Nq>viOqsDo(24YCmoNGSH${jOjS z{*g>x05b>gwKxYNT@2M8!}9MEAjg=u=9Co(=*wJ z8~{)u*OF5?#zlx<>?s}Q)@UlFguqaD<`K)Jjtd2USQb*=m!bzxZGqKUO(0ez5%!Y* zF8&Q~s_WRK=vJ@qY5=^L1u>#BKsk`J5neW?)1gnvMi8$6?iRO@aEG=W0i$!W%7oPV z6X!K46waVOp+Dp&t2Qr#=gKX49``o&@Y3p#{vbd1%^a+h0;*)!0uY%DJJ_*SnAS*b z=vgZt>nxtM5p_^WA;Vwx`rJc?IDW>4{N`m)IO;L)CSbcx4xr0(w(Y6_Tw|SqO@K&q z#6}8FIDTgSSVLh){+K6+_GBM~#GD)@XT9z@>OxiHnH06@Inrduu3z&^*T+PYTM81% z%lR5W`x`jpz}_bvzWzanYmNu=T*v_$O=D*|ULX$cQ=QX*n-IqY+kVcM;uIxGcdbK`@=OoTDok8p59!oJV9m)(iq+UL8J3y?W7+H*jt(j0hI)M7 zo@uHACnZbHWDabNGqoMybFj$Fpzvj^ZH*GaTi|`{@HH9L z`{$85j=>4L{wG8S&ST|*1)PRfxKivqzR4O8Yaks+&r9#d!Us@1rZ|_J*0D7ZGu6V z_OL9~Z?I?x=MV3b@Yp=>RAJpTVEta|TNwX``jM1t{jIewDE!Qz1*vm!&qWEatkr@g zS8r}1M*Om6_xd&}1f8AVmTuq!iuvy<;07WOT?ki3z+-$N(HiP?*l_C7fGfMMvqv$3 z0{e+d{K6~A={P~#F{Y#A<5~y;cYegHU!Fk|U;+zr1mQxvLUsgnjxFcAW`p-NZ~gv& z<;+cf)W+T3z?oPH%gduehFHoU z-r~rFAug08CSXr~B^=7gC6XWHD+7_dgfObBxr1{S`*^0(erU4%EB-Lf$OMwe%r#5MB728l>yryZ0IhFs-L z$XVj&*7{zl?(&0Mg_^)xGWjT>M}3EhlwH_BNJ1&bXz>nkZt5s5-{NaZ>ieFsN=wNfA4Z3;uHM^VbP+7QPG-)#|g;0M9 z+uY>$90#G$OP4D(dGf(8A0XOySi%h44*>hsIRl)123R$uHBaB}pwwzkhl%pzuP>ju zh`r0sIj?U)Z_juJo{(oE&}2%jp1`pw3Vwlr#(k!FrnKEk2_N?+(J$0Ys~9GEtmTF> zGPU(ki6DNDsca0`#{5(dULYsPR~RcZ$#0c=F8dr2cd~kCs>z)GI4{vn%~fQ;cbc8H zyR89ta2)!NHDrUl3dEB~On#IJuT=|(ZN}*G`dWvnJbgDLvIEJzQ+d`Fnb7(nU@5Zt zkxfq@ZfkXmTHZ4$o{IiV2DcdA=60-0aAEbWo`rnJK)#HnicW1_IhPze^V-Ai16o~HGC?DwX_FQ=d0SnD17 zfdft@5MP^IX2Nk8_qX5@dG*PF>Ep~~d7m61y~5nNu}!=U>&bbur&;H$Mu_Z@K{fHeQrl9n_%@i>B@+J2$F5Te zZG)(Yul6lnp;3}J8`h=Xq0`+*p#<=aB@_Lw(-0TFgF!fjOMz!lrN*Z&FeZ zT3FazM<)PJT2%>lddGBHr(p0RIMt?G%>d#G}$HmLtWpJ!Pt$=4%{LG6) ze-v30{7}0qm~+9JiR9?F0gKPWaL4F<;zsPl4;oR*tK`RvgodrbO03LTou9qglszd)ZK}ev&<+%Mb0k8)l`P0BWltUcq zBa&SNff{Lf8e$-sD0fZmw=Pa#mIl&5`xp97G-3~XJoJBC;SYd>-0UGz21}=q{H$p9 z^YgI^NX>N0IKus(1zeFvwRj>*kBjI)jSLT!e|k%Z8aee32Q*8ZgCiG|Qss72X20R2 zl9*5DaUPe^3KNu};vIwc=pH3NG8CIk86ltDR=;d<9X-wWSBtNThi|dRHPN$_(a*o6 z`^*I9N1n*fOZ`Wd_3z2Ii2OXkm(kNV|AiEMD17kW7=VL0f*ziQG@SdZ-e*n&4Mb|_ zzXe~%4BQ;}FZO%*rFYK%h{|UKIqA4;yWbohI2U94e$cD&?!MRw22HW#U(;#ef;Z2OHC-4`2ro0c z(bt(XZXinJpKxuMKDfTrxGO1{8QKav|3{!(6OaK1U;+L#e`aLm=k=K zn^^U&3j${+I+agfEI)XCq0E8gyL-S){4th~MG2p5w3b^^ss+Zwdytbe{(XM-6P)6f zxV%!Yw^Wp=+ZA;7VKC9Ug+L48R0tF%GW&L$PyFt4LOo1=V_w-XvAXtAogPPLY5d#L z+Nh{jH(TzY@3DOyPm`t1I)9p+brANaqL3GiZ%ZhzOX81_2Dzc#SO2BkJ1i@fcb56% z^rMAbQ=nVo6N!P_cB;g^=W70l+(!do@C0|vz)p_A9~Z?pluwDLi>Zkar8_~82DAWT zqk?|PUi;*kB8eTsX$e?nSrFcgHeJ~bKrvr4<}K3+l9IEl8+~`SsQK`~CEWa3fJ|lp zN=S`o4$?NFG^h~&$GPG=HP%f-d*{pUAeX?A*sfI6YB2_ASgswey>1H(lFPQz*Xt@9 zNJWqCmi|X*&w7$5T^I?f21M5hD+0Fcd;=f%DaeyC-@xnE>)>ljnM@MK^p> z(YN^tyDQX?A{oik0YD;ISqWGKPqk1EyDl@w(}G#OscIbg=xhyuDS6A+s9&K&?_ZAl z==lCX)lora7t`M*2}qV@%8zc z|4!F)2gZ_n!Xa>0pA2`T7qqNQ^3SmH=oPjvq+Tc;##86n{_Fh6i$WFS&cHbDrRk5AqYMpJRLb4vaCBk~o;Y$;n)+$z_M?bqB1FV18&w9?j!ZL(Nq7@b+ zZW8NeXiH_6eC8?h*+gDFj%4ilH)D}bykvg2< zBEGlBNy75FI4C)gI3>je{f=OOB*-M9s{Bq<Nhx{E5beR7gY03|gfoX6$7K-vyCh&N2d|9*+hk=f@f zp;#pp+edjNSOH`Cw>|6F=QbP79+D=hIM4mn;yM>G0@LD`v(Ay%O5VWjQG;*bIJ~)_ z5=wv;oIE0AoR&UiM?JFF6FL?chP7wJdGuaPgnh)@p!)q1E}K_y?uSxLM2%$iK26EG z$gzr?2KTj1X)x|_`Cry?GN&L0jab(@*Dd$Xrz8=d*w=-2*ynTT@&A0vD1`v6HKZWid;WS5xW}>b9|mFTuQG)gDw|{ zc>nGsPQQZdse0mW zMzcUqNaw`2QY?GC@+v;qlTRW@n>R&e4}|Q>%Eu^}Gt*ziH$(=pzVOdt6nf`a6}Lj` z3W3V}p^n#eI!MtLh+LmZH7Ari(gu$du#(79HfFivLqbCqCPc!R7rN z&N`UbtE3F841`fxA5NQ;dWPhZ`{+^uuSL&%b#V{epI)%)@yBD=))dQkyo;L5nYWQB zw(wfakzk)zmSn*tZEjd7)?R-QFRI4<`;$`1Iqr5kR)dr>6=8q7ymhZ!wMsFu*mYea zUtOK8rw>yExUEaH!_OI|+Lcw0zl^nsQ7669w(`*fM7y{)AZ_qwdH27Xo}y?Pfl<)p zp!~Wrtwtj9Oj`1Vj-KA6M>#+9!gdBOwdABiCV3@|dLH5~P3V*h3q@VsWiD;^7py|_ zE+W<7IKb!ppePZ8!#_6>%)L_0LfK@utN{O?Qf3Kn3r?i0HRFzbCIu<+LjW-wYkb<&1;;%oe8UTBehA_Qq`Zqw?~mPn;U z+0%27fj3*PoOxGxMJj@;RxGAq?VWyBC3IRy12>D74wcIGB#J*}&)$ZUvwJYmNihe| zpXh7EQisPNPx)$+a#!scFIc8~Gey1{FIA{AQ-=B zeQ8~7jpq6jXFf>s5J$Qz$}roJ=1J*m$+yXHNr9M6a> zY4J37oI-G>Fr+N!V6EeooAAL!OZex;6#I+Q{;|I`aBm~e>hY&i6GA~+sXD1qTH=x0 zM76*G93wB0FPw%i4NYkeyHK4)sup0*$GvfhZxF7s5kOg4ceYq8gSU@Yul8O|!nl)Q2gF~wpul|DAv^HNox4ITDJ;(gr3yGBWbsk` z8!SFI5-|{{^x2X$@@@{C-=HMI7_IcjJb_3Zx`|&4-#+hI{TMo?Lr#m~&X(I;xB2Lf zM6rgOVvhR%JNUL{RSKD<>+%#D2fy9jd~3@De(%f0yGVO`y)5H%1&Kl0 zud()hoKBP;k#Tex(&pMeo*?k0*8s6^cf~FF_|00`0bK-i73?@qxrdoeuYL2gK$;Js~rRc1?v_~jEfgX2SI$|@{K>S%)HSg3O5FH|OO)qkq< zeix}k^prwpL&(NFo)+9W5PWRgrRH|B-bQl#@DCtT;_t6!mpU&Q3L=oQjvPHX{nk;@X%-~Z{;MI~k(a#m$BnDV;@H`~DZqPBIgz|m5O z?0f!J>Bnr8*2eC3bCnrdp<3fK(x5wXo2TebKLTN#IplqGh}$01?4Uvt*bS}oVL_!B zR%Rxbx{j}6OKNS{npA6{6wJcaWx4qi>3+|5h~t~GNb~BvPX*&a+@asDO)*AO;W+eo z!mvfSz{UCEGT94hu@?!+7Xy3G6{-Y#RgP=lw0+<06x9=i@cs?ozj0e}V%l7*J78>p zz3xlS_^EEcJ}~J}>17@N&17=YwP|-0-iFP7N}1vlp66uo7PKpN1vW*B5#u`_V< z9jwB%khc4~X^EBfBcpeZEQ-?}Iz43WIn2c^Sz3v*P`=WKFP4pTT%AnI)5*Ppq0fpN z5KT#g$EMB^=#iTLUFfS2FN+kN0-OHq3LT+1HUFLJV#=qMl3wQv&^31c7Q(S#^fYAF zs*TgM;U_Fk%eJ?PpRf-VlQJJJD_-R_`nWV4RfKwszS(1E5E{5m=d0N?Z3i3NCR2*m zR_WleAXB7}vdM=ulEjscun+W&Vlfc2=&J{TnA#6nV~`@QhtJJg+#~DpXC#EeRLyQ2 zwysb_6Uq~_1M1wYmxm~To|3Z!iWV>%sa*5SD9 z0j}b*xxZqsNXZocIg*g0&HqK+dqy=CHEqKxDk>sYP>`kq(iJI6RZx+R^cJwtA)uiI z2muvQ5a~@pKOYav0*CyfEg^(OZINsnid&R2ODo@SewjA*%setIDV8(*t zq{xx>gkg1t?z|B`yCD{P%;V!HGnk|>Zvoaqbcm%^@6d*nx&D;h@`9(+a%ru16~2(Z}}(KlXJWTh(1db+7jr2GIi zIS$jEMkOh~f7C?{HWV$Bhm=~8GUHw>Ey252;&x_P^Vg_u{eBLBjNO; zMg8Gor|8O}<})4cX;}wy-l~r5ZX0Urm@%uLzs9la5pdGww<_);)$*~I?lJKQxb(fdpZeXQ4j40L&(4?2Kiw|A zomAH&1Sc4_v&Lb&mEgyAQID}qhK#o0NI9+vUhPPYZ$~c|kEPNovm%AptazUg_9;{5Dx zJ+G6eF6DmNmL6YoB+SNWN*J?nO^Qrng3XE+8*op=ZTktq8putmz*T9dtX_}wV&*fmMhgB$y>;zl5YhZa5~oVN9=-uW%*kV zB%#l2vhN!Qj)L<_j_;mbmI^;OiFqLXKKjw&M9_U=+fGt86i#SiL3mK=pCn0|<>ikA zYr)Xp`cJO``^h^}*Ix^b)f$n{ZCO;0Z;(0mZcj?VDV&7L%?P!oIv9KNQT>YH-?sHt zBU19h#uKLa=_m~Z&X@&@-q{rN7&HBu$JuRMI7y-jM|-}sn^`Y<0s(CwLoK3WOd2CR z=e9N~y<;3hKNR*FCECC$J8j#QvxQTWnE>rJ4gXGc8!WQ$H%~`S)0#`oPA9Bh9}GIq zNW;1QN9d1}b-g(hpWEl2WB_vce}>Hh1mX|q)`49$JTmM%8<_qXP=t1X`v5P73%uv- z!(g{C497$8!p*eA3K}@jTbVo^h4{dG=;y7H7Q%AYAUETK76ooq!?O%#V~(6>DO!;qt_1FsVeOvubQeiE;twgIYGCfiW%c!eN$;xVWs@rW0FZ~5uQ4mD5WZZ1 z)nwyp{rK{hcedi;e}_DOidMbfrJ^#fYP_pkrq)B%^A5Xt?g=)|Z^mHe3j8V!G&n)p@+5 z1b+bXg}N%V<3Fbbv9%+FTzuS%zD6MKQFP1e4c*36teGj!N1V1Z%WXUL3wCu2Om1oy z=$S?4Gxqdr1R(d}k>5*_6^%s9ta+L<lsiQWl^z5@zqr#v>AA&S_(b2ddpsB;A&zsdAIVh4WEZb9Dw>YqY5l4&>U& zCgfuUGLGtP;-uP|Bt%1XG|?p7w_uXJ{$ha%MTf_RU~kM!YCCtk*Xf7=+;?xk)ld&C zr;iVw@z>Oeka$FpVA*W$p=Z@ZCvC2Lc87dARh#yX!JZ{{G zJpN}X5m>3`S{W#=HKS{(%z3$e-WH1IvS)OB9cs=Q$t{iul7|&MO$-SwgvJI#0ee2$ z-9HKqCc(;^<-eefG z;*pi24%m+9uQ50&uNAMF2o4s9+eZgw181goyESngrf-o9xa%+7V}jv8firBbMLQU&GMyEpIwW zCkK&;uDWtnceVe}kp8ea1qhgJ>^9qbQ$_6V>N8N}1Kxlbl)6<&v}(iG6|<&0YW-|l z$5jMD$>uF~2i--`kMAgXt>BSE=G^-x!0b+tD~SE@({^!PS0x7sc%Xm$+14e*JMHP` z+o~WqwG{-Xc<%CF-4kxk3KSkDQ4TXS>n5mKhQ`8;1Yw44VVJC(e1j;gAw$j>?vS2w z79Tw4$Iu5zsrj0hD7p;ubU^k*qmps_10aQAB~H3tjzZ*%?7%407e&$E90lLTw#eKm zV~hkOH2L$-z<3Wc?k#Z4PB#+~J1`^=+XT1`S<^kRg~UVGTQ>1gFn`hIvwfC)3cY;fyz`$i0 z4U*PLk`oCcRTp;SrxW$AO45#QUbY~p`^oW|$-pe)YoefAql|Ffak&xYP=JA4|02r^ z%)4xAAFAwI_Yas3pkIbX+JKO2BZ18M+B|QuX|0wXF!v!%zUk0NM3$}?t-qdei~BKf z1`u0s3yVDVi;lx29ylLu{g4 ze9!<-pvbBa4A`n`;sz!}df_XT1uUJ4xmNtu`~0;Vacuvt0i#*8yTro9eGmE8#w9Qs zVXdazUJcoDJy1*#tsQk%X)5#a)j&lj&b7pRI} z_ah)>`#;OSQD|Vm(|0mHhH?I;Jppw3l6Ct!@C? zy|H#p#J%%Kz=Z@*gNV$3(NOZ*XD%Pq6~3fmR}Xm2B_%J|-Hg5w_FINUUz?DSkQs}j z`rS#omMswFEP*M1>6ec!3z1(owiU0Q<)?wJ&5s?9ieiBj^sO~8LN&b~V|C_s)!Nx! zJ_#R~UE*=%Kp0S-Bv|l~Ydc+Kvxc%-;hBwGL!>(iSm_mS?ArqGhh{GD@sMJEi`FGs z5(UIaeu&7m*TJ%{xiFQ)$wD-KP`vrHXm2&oTRc2z=5u#vc+W1^aV{y%^ceah+Kotn z`goEx7v-T>XFZ&)tk{B#HMJn`Q>eF9Q+D@AtJ{iE_|5o(Zm$gFSWCk{M`TM!^&LUe zBle((6KV5jtD}3FxIVrfqXd3df(;W-(u^Y#d_nK&5HL7^blV5-T+<+~;Ct^M+6o`6C=R_!DMFHKd7==B zhLfZ@ok`3gO^RaQRn}Sq&47XoQY|FtuT9=v(cv}uy6g)Vpl(B*fy}l7C>axlbZmVd zvvmQ4qX^}YSv|fz+o${RoA4zIb)5rqqfc_vw#Gds4(^u^>O^T7cJbLFAo8fP83VH= zyYljN=_~HB*wMkB zt(YoY&w~VdLi6dKCM~WSSFNIJhBXsTiQxUnJQNfQTpL=Hjq*LXF{{Yrv;NnMW=mIu z{fn$;J8Zp|tJ=y2lNH4#nePjAeN%)V8w@YfQ5lnXKQxfuJ`@#c;cRg#9wQ$4C<+UD zTWde8^DFk|jDQNkuu7~Ui!Jjum`%p>0Z3^@QoB0$H*Rl^aLyP^AO1<4n|0PZS?^w9 zgd}U_KyEBOxrX1Vs5WbFQ?P5t*96Ckp(%pS<9DT!B3mFS8npH1YZWx@;KGGrsF&4Z zwyIlx1|dnF;w5a-D)S_u0IXB{tr7?AUsY!iNjyt}Sq%zXF)l$GCo3ksDl6p7Em7TW zvRtQVpPS;FT_Df}zZ<9!Nx1ypVn8A8^Jmq}Z?b)yhXS^hmGthOhbl!j(uW}34sv(3 zaV>4G-V=8QtY=W@3g4vz+=>$u?6|@H1k1dZ(26NAUO7X#94g$X1YYodYSS4O<8w0J zxopDp$T;1l92bzf{I9ACqso z3-q3FE)4wJB%S)C8V}%|5~o(tlFosZpY}m^MLYP9ry}0BU+h^ed4J=s7o)j>#}$8! z9ChWbFM(D5P@UX+Vu0Ba6HM>eZ*_bYEZhHa3V3A#o88-pDG&x)lN`oaC(&|P0{$k< zW&{uK5p_Yr62H!ls}8ox(unYn1=w)2z7+`y*UNKv^0xBPPXD&es^UEg3b5NQ#yVek zq+eDGaJYF1=ZQga1DVS)Q($YD+s^=yBJc4RtKRw)7i)e?X?NQKu1fBV7(~VZVA(0V z_NHVTiAUf!b-BU$yZxQ_F;k0@gR)g8`-C~v%oi=N1vw#mV-Zkn*4UG8h0*rDxTXMUb$-`GsQ+%d4>?fjs}7eV~_-0;GMe0f-S=x#sk33 z@+)1FBW0&7O17mmp z;azAqDv=H~#)#1@8#cg5=)Z2ukw?IgrwuH!$Pd|%+r>roE0evTkwACgquu}E`W?N; z)OUH88pVshw-Wng#{P3QnIPe7EdlM_Mtj!kactA0y6VNkjfUL*JAgl8b63EXr>fU)9~|SRguiI7gnj$lRsSdpq}; zhI}a!&z8T)eG8KNI1_JCV7NzhrFphGc1W(*M6Pj4K5-gaj#aNl1AXU_`~!ifU~uzr z2ul)kA%d?Pbdlgq@SeBfi21!P0czLAW;@Wam3#Ho7n>j}X+YiCHZD~P^0y*h-6LZf zBtH4NU|l}UIVbU!qi}46+=r_{nRucRcW31yQr)Y-)G{4A&W_1XY)AJb^uWtbs%WWmTGgKDHNBwgS0s~vw@R{AMm3(F21dw!i&iWCUN~jQQ8X&l z(FdnHso5e^Ek60Ax4(b>)iha7ewv~qK;y<_QPSKieBzDuDCO1#Bca!a55It|K~-kE z$*GbX7USPuJtEg;6Sa)*#tyrAn(9iz>oRm>?Z@5qD4zOp?Y3wnc!s>sexh@clALag zMo+P-qyA$jFr2v$pYTb#I}1(hZLzrXEDEUTeMymVf+L#=$InJ08^H>t5*@p0j$M**jJGqyTh)5lX#ar7}U+qH!)X47)WRkWFDc3E zY){H3bXf8XO;Uv-3-d=VVg~?z8TY7`BT!1KTvF?!u9GXIGnmYI;kov=c0tpk%8$cYH0}* zZU{y659*e-)TFHV6r<*crUJawPby;E5otvOw5Idn0+(lB_X*LU>$|0irviW50kF-* zvP(qoCu>%V!Gv(}A}Je+0$`fpS~Rt^T4_WWzFL6c(pRvaMv-9Kq>t)n#-g4%c|i5hOG}^?j1+(Sh@j($%t6> z2LOC4!|V!2(c9j5@Nl(`7PADFC(~2dmEV2L==rpdKhw@+s-Q6MDxbL8`d*E&BvBDcXeld1x?qm!r7W{6Xz>ZrDaNvfMtplrNkpHUlx z+s{V|!-_$att+GgzTeGH_Plks+#Z-Hjq70_d$D+DVf^9`4ZS)~MhRI;Vv1cFyEmfsO`f0PtK?ocm3WOoFB z*Vy8@``DV6tr}b5{NrhF^D#(m@u!HvfyFdvZKj&q%yaPaP21#EQzBh{IeiAJN{y7a zZiwwTlxLYg)I-Il=aY+A&-nY1DP!-0bqf}Cn={R)iT;uIfC-ikQTIs~)M**_kpS#9S{S|b)w!1VoFQ+^_(h$L4`XYmR6frA-!Cx2SoQ`VIH5n}p8 z`wINb+J~Hlo2w}LzNuRcX$geDn9qVFo{Kgdb~&{e47(4?u}2P@?BdyI)Nq9$2}OJN z^$j`)QinyYi4DLp{$I=rfASe5FAV-a=3W4VT$xAxt3 zz+#(&^Q)HV9VfS76n2o5k2d0j&crVrCOlH+uMxo2W}_FscoBRv-E~st-&MxgM{%Bz z>vUh=7v_wspp5BN0PspnZZDarkA69*huzkM%rJ*={d1Pu*Mo`|?c%PhSRGAVg|O@k zLCqxhvatM={A)FU@t{Y62}`-OS8Z!jpe$cxZxM?w^MVnpaNx3yA!JppBf)t-mBwY$ zgv%h6l)ff?bu1qpXT`h2aL&l_dD+%cPz@C%2WhC9t!NYhlG=(>E%2t%Z1s52o97CT%jj0tiJ_ zyFQ`W7P`E8dM(Isim*#ldDo$;0Tx_y6k-d*>$z49aMgCNF~>O$T-Pdp;FK-{!8=(> zyeyE92H?&W_kd-QB3vxgj4}lXlb#2-ey`N`xcI^Oc3hJA?PktJ9VS+9qxZH#`48B( zz~Fltkzh@_Wbvpq#vQI8!r!e`=6Rl34LMiI?w8XmL?aTsjwWD}gjC0xXLC!;bz7!8 z;-YnNXUpw-i*1GK!8Ts=`|>~Si!1@2p_ETN6}e)>^6@kXQ- zAMNu!5gUsN4TntGWJRMYi?Qpy-snIb>+GrLg;Srj>Hy1{clNg{@Ozd3+*V_+t%I%N zNs5NC8*W`>XUixA0OD}8M!#m(MjYSeh;`I>9d@GS^I_7fmmBrOktZAHO~F)v9k5xM z6JA$e(-g{xCPHd4O;y`GVl+NMDKD9IU8f=6-zV>xNRg~Vh@46>5SCr0UF@@Q%=Fbe z!u9yhOM@sn*cIp?{)quBYiz;}gI?eiwQKGtIh-ykD;3kdEBejEvanf*#XKNdHc!If zz(9`vxVSgFQRQ`<_FCifh%;SJ!G=BZY6dC&xq8I_Nu|}Rb6}~r zPsqTnD2!D`CL1#Bnt!u@N!Dw`R6X~(_%s6syhLGcL1v32h^`fo^V2P?*t7OPi!V04 zP+Z1}G;h>g3+2WQJ~_>iNa_`%!F;WYs!n~FzlGV0i3Bf`E|*)^cFGMEokb}~JxnR` zQ@EoP)ah)`Lf=Ht==(8acIt=#;q&I%AJ8UQ`C7M9#jVL(%{7O%XX3Oa5;sQ2ugG6K zT)zz}D2EaQqZ(T=i10?-xsx-Np8__aRprZ+b6Rl^5430nY~aMMQR7@W($m>(n}*Mw zV}ebv7#V4Z!RG8N?#o&~V6f#e@VMcl@bTQkBxP-ibSLuqo&akV(eJksrV@7v(7w81 z#yGwgzX9uuzfvu2`Dt_afS_>43gz*6omM}kPab@Bcd603iB;}re=gt-rr=8Y=c0-l zElQSw8`wIq6l)Ws9t;7?WO%`>qUd`7_nqJ}nKDpoKmd|08k-2US*oTeogMSExSHpm z2zcb9>$E6-NuTzRlzF1COsG4R_Dc+CmXF3nw$6K!wazS3uY9h|s97r>?Xy07k|uCH zVH7>B+0uBVbaiJG-3%;BlulQmye2ue-m>1Dh8OH)E>@-*V?9mn3cGbd!SLlxyssar zHQ|d+zT%ZfpHmfdoNdNq5gKu@bIT%(c@d|>Papi!U-fJaL@|XMkuL$6D@l|SAHXpF z8x1g3mh%Q#-T0Ng0z1CsrB!cEdNUaeM!#~wgK2~0`j#lV&mR;T5ox7iqauiI(9r5( zwRw&<(aO4YBO75CfKjYV=ZXI?f8sf!B(r*YLAp9WVrv+lTn~I|R@JC|HgzN|F-$ZI zi85lsu_im!zyn7jOHLK0V{WvLp;>de`+OnUJ^d{0yzwrx5-h|p|Z~xH7z}kbr ztot9}_a@lR%=8QG-bCB~FB7v}flRp=m8MI79ApBcBA^}piAX;H9&ZD?7H$6bt+m7j z{k+R9YhgjB4wL{n-e7!m%Ma8Wff0=j{XE}1a`P$$9&AazzT?^Q#338tWatD*w@R0w zb~9>CH1_yGzF!pgw*-H(8!|p~g4Xh!<%@|g_)(FSHD$0I*RHzBt5)pcX>qpbO2-X2{<$=88=623ka0r z3k@Y%f5A?ZgUwD$bgU(^<8^4`?GMbXEkrt%EO8s8}&?swBXN1P%JcCsXlS$Ec>gJimB(HlsG7RL_GtenpR^~xI(q}FVP>u(i{q= zu$HDhGxPK1HwMntMkdg|YJYCijiQ%>drChn^V)k3_VLWV*Mz4)6Gh@%S=vT-D5+K( zt|4HaxV*0I`FL9Kf`9bhG>-I|C|cQ+1x3_Z3oNh);p97Sg>7UVYi&>^%`vxuN;F#?g#fgt$Zgx+j4zKTqCl1t8vo7b7~p6)2_8l z-j|HVmWB2F(MGuEVE-I2SQ|ypl-cE5Y7L(K0Fxm>D0Po{7JpZa*=c=m`{(S5m!C_` ziQdAz>;B9Vh4B3O__B2J(P$scW6y(z5nU9plX8pk^-&bUdRJRnJIkfpg#nMiWLt*l zF~~^c<2?zMFz8FDycx+kx7_t$`el3K<-A6>oMX|7SF%gX^;xwOo8oOb0;A=kEmbMt zH%QbdvAu5s)?Dv4T;UM3jR-#3=OGyp_@wI^7{+A&{d0Eab+jC_7^=^d8Kvwqv&Ds4 zJrh|jsoRJvHHi3nD>`aZ8Q=6a4JGqINfdksxTzZUp)j} z)(^Qvag!b*zDMn^ROzaH%}1;0I7a)%{ZrF~itewi=rly^pDI<+br|aC7EFVmtaM>@ z@^I(I>3e!F#nSC(uYo;kv}=u?zMF}nEC1kAu}$5?xIqJ?8kg|`>^RLezhQ!{BurJe z>?jE5UY^ZPK-@Ovhq@EIFH#ifwGJ8fyE#ZauTJvOz%*3Gwt(;FIp#*tKkBBW02FVz zp;BH62*Ez_oEY5#fvf!pb9WC5m!ZtafqCk>S_K;W(J- zKr;K0@(iBFQcKTo`~7!Cvn3NpXXXkxxBX9JT}e^0o{Gi#zvka?rSGwKfy-8nm*^wi zOTb(2%41oxtc>8#k6kM4)tA=n8@8Vl_CCG`Fys1jlPLRmuK< zV^>HKE8m~c4S2hKWX43NCO@I@VBDRnn90@!GQEznryR^kDi!x(rVB3BBA3dFG|gfg z?%SKX7KK$nC%o2FZwN+QR|P9AHRkVQU-k)-seJc{^Qv%0q zWAQ$3jLKGZ#<~0ltogC;0`stIOKG(wB!Mm>ZZ?Kmh6?3J8gPMf#&N-9U=zB)(Bw^I$QsdKe&xm>5eY5Qpo z7cK}kMPVmzOLhv0IZ#^{ODErLYndJq8D>V-z!UZh1 zudouA==KVX?~VUD5fa5g{@PV@+H{lWtwEhwHY_+u9Q zdYEU)b@bsObAc_CpAcFgAR)fx;4Cgz8ff7;JDT{f+xH0b4C zd*`y+)6kYb5VeE997ycho8+2sIAvikfJlLXNuLv#>U?@l6v3l5iy3FoLv!w$%t;Kh z_=#;z6eHt$8MR{Ft z@}2h0>&>Z^9GaA@u5{BFNm^k7B5yl}jp03Qn)v(CyVWT`gZ1tgnbiejs5WUjh3Sz8oMg`OX&c z({BE<{VJB3)SUw>gG>S?`W+eREQ(y&Vo#1F9W}hpK=8uSlTw-zX^Vibu1<92r~H$u zm#A~XO{rwx=_3~U`^o8khs0^VB`@%G+zG3^`~Nx%)5ImN*63X|Anzxk(6jLwCG8^Z zaIsz2voyrhMG}uJKF{8Y8Pj|qh?{M~aAYMrv6G&h-bjnjG(C;CCO5#S`9MUo1u3Ql zY@xeDeuwk(F~G={RTE4+`HOI4ih#_4yV+;z3LPpx9o7;^Dzg@mlsp3F?1?3uuQ#|w zwdWXJP(&r=9KlO17aJIeiYKf~t~g7JmUh?~JJuSZ+6q2kCCqsQdMx5^X^2VM5P-T(W)?$=M{d(aCUh&gnw6kxf%rM+|nug1bx`4`ijKr89z9mE0 z%XTpGZT|QjY2tqz{U0t37^B~Wn>Kx;r6~unQKRq=uY2>rXOIsj-fa53|GP;5+p9AD z!>6xc%^Az>fKz|P(fBvLyl5$mN_3O0TNU)%8U|uA8+umd;2&hXWB(?juRR9>`Dc6! zpuw~4MWudqM;A{16>;12797IBU{`H>EP;ClJFM%*OV9Z4C(4`Fnrt-h+q-!j%pZ_U zJ5D?FU(ehxfL~kZTM$|tOPXnZ`ql`{7TR5jEm(yCPr;f&#IGElP1?ooDDlN(p{DkG zvt#Kj!Q3wYC-jc37#~XR?4<>@0`gs*~etcOa*=x!zNS z1hBmLXAJe~o*&N#wbTxw`%%s3wZ&;?mY3V1eSZa!9790gP=yQNYAJkb3ELQGcl{rJ zwFz#ihR6hn61-KdtpBln8%mr^N zNdBFG^a1pz(~h7%o)sO50n#$Z&D&c3i+bjiM2sz#=k}K_y`5i1^dPoVUQU zw|$?8q({z-^V7#_u#IEE?Xdr*Kq0pV?8b^e$_6K`XRds?E%)D42v9<8Og{2sR9=yk z*Qqz&{hd6t$s#SQkj)1+jGESROhh-)?Vov6o68VD)k}(wL;!W;DNIXOl0hi*Pg2z; zne4O=5{w7ZzJWdU{+j@Ci61`)PBsyo|9c6RR$&m6kH;`{^J7@wWB;`SciWAJWR7x} zlGZQlQDhgS{&CLV+ETG#f5YW|u*|3L^&LV{@;A&0TG|rH?c9IRp>0rCFs5FIv;Z#5 z8qkPGFw9K0XaDmV{kos+Y>zj8!k&PCthJcwd-Ky3`iLGj_NLpc;{%g@)+Z$gD4V?{ z8=P|pd&U3~()KH#K( z^$9CU`QZjTjQ1(BVh{}uLCYl*_uqo7=fSZqtNzfkiV{L*Jp}j2KUA@Ee*yXlyZC3Tr>lWD zeAZ(f%2=PvOa*JF`NAG!SdLwK{iM5hy?urja($^<%sxZ8@ZHT1475b({U5Txc5~r$ zJT8@<;5qMIR4MSVQigYnY;ezf(meR7Jvxc_rH-2*G!{SDgXgd^O5v@%MwyT5bY2!v z=LFk0z|!{8ohxjZn7~}r76@xjuOgS4tAJ2&+&7%vE54>-eVcqz^h7u&@V@r7FI))@<#M2|b z8H}#H<)pps`x`A&);;nfN6skfi> zKk{J78IEkV4Ad#=jcQr6+m$o~P2#r6n3t2Ceo1I1jDtN?@!!RR=Hd3{ zz4fL|(UJ>jh6ry20-0O2fH^$~Ymj^vNx{ijkV;mUQX~q(M zE7_)V$henC{d(*F!y`5fqqy5;6Wsc->U-@DQXE%-U%9{`GWUSnr@*ef+M{8W(2>n& z_AA-M*(_5J3<>;5`37BwD8y4zfKKJ2WPfk?V}?v$TJ*mZK<>Nhn)v(R1B;wv5HpGc zgy>rA<|94+(SY(Gg&Cu61127+6@q1uLKqv|2`aLq zB9t-erAe3lGJWctTGYY6=?6%ZJ*e3MVKDu^_6KaUX@>e}({W&`{4W9-Dm#VnZU!xg zB$!dn|8q;IN(y!tDU9-0&~Myi=9iV!6e+$x6o5K$2idL*Cj9+g2B^BPYaMd-HYzd7 zdqMvM+)V1YuI83421Jd^myD27R>Nh6U|KyAOsi+`m|?*F{~_^)Dn)Pj^_=M1w~ygD z%d_X&m1#9T>%kmpOYC)x8?}p1{no#RkEQxej)o_u&ZiQ6mWAsc5=Ro3H3>L<|~i+RakccK@Ms~`ptFHMdRqraLjf4^6SJN9ImpN9OG z6GvmPuMfmwdzM_-?-kIOrsaR7%rY9!IeJVPJo|%Aq*=!pavG=9aaJAnS)f zPW?X|zj2i!xqHr4k5VdxT99nNhOIjuMQ26s*M|j83e4qO&6=jpR{&{!dT9sHV8fv0 zdIzO7m4#+ew0^W?iyHW^;`>Oq+Vp_)9SeyzVEf&-|2eu`+HD;d%!cN%6yw#eEqXgh z>tKDWBSvK*$tSdWS&k{VNyal~^vc|W7&`iV)51aFjpxa$W=o6iu->uwoK)ZYyx%%6 z^W3sNa(ij)e_DUfqDL)Yvn#Vh>9#0FOJL#B#g?_aoM4|h-@6%sc!v)ztK+_NV#yXf z2dsVOb}YMm7FKpsq7dTh_ z>tpE)>rvx87!@j~!eFdI(;)9tstIyZplwbOj6#a18em=|E|0xguJy`O)707jI4nuGb{c&-uMr%rkwKLnp28sr#NjTo+(HEq|~vKJ|!X zBXfI`+N0(Y#nz?3c$2+PaUz)Ep7&_U&P!_J$|@wDZT|}OZCK!Q!!bD{=9MEePOru~ z7fI_qEMF@edf2CD-jI%ht!zfxJ5=;sz(q8p-H84XEs0kHXGTFcTWe_e)N6#~5-Es} z7JBy|bvqhMH>^uv-P5urqonGmHb3ZwQBjLoBCS95U&aTV?AB}#<*=iz`QqwD(K|*z zE!>+dXvYDu*ap^FC?8G#Ux|GN4kj(t46A$VbfmtO0Lz(P?M?;qxMo|u*&5fJmgiaG zg1?Aowz=2Ne=;OyCOO#ngzJFf{82Pu#o<95c%S3_cgaDpK#LFuxLz!4V{{e{xZ!rlcMDis!KLoqTS3T5Y|602^C~bPOA#noX9yJ$8=RPWM#iK!fD;_E!-^Ej<_KlLCkSXd%n)ta8g6`%|$S}d#%Iiun` zYT6!_7yz~^t4n`3Vek%J`|X5)b1LTT>{7XZPM@{Ub=+FFUN{S>mhIpD0t(;X15Br= zTeXs@ez!k8n50(Z(H7V$hp0Wkv^TcghhS+}RA2_>HH+C#8264l z`>&90z9DKcV^Mapf9`TJPoHzLst<%#9f-pBUJ?L_gy8g*uU>it{hHuKQBPj`awCbK_euhSL_{ld`7oGygRa9S{CT&QRd$Y zHbLsZE~s)7?xrCr7ElMQKreiJxw;1lkLLlu%z>0?-apiDIU21BPJW$KfwZI7png zDBA@dbKt#*oeOh)fkaG5LX`ogYhq6m=(=H~+N^`pLp87QUrFJL`$9|__%Sw>Dxr;= z?<24IPMUjZ0|J*s4j#>wAY50sH1s$pC z!ODrJK-Li2ssgf`4s8J$7!-||27|BFCyk_@2Oe;{&);_gSb=C|0BGGq8yniYWS9(E zkU{caAqfcgdp)s;g4QYF-rPPtPp@t2-e0kaVzas*>KD3Z`c6PQM=g-5_khQIGyC>k zm7Q8_Lgt68o#gkqT;50T?V*#2wq$haJG*vBSEfds7X^i$6YZiB6kKPwK56Nt&oIJS z^fM?(jVl&^ejiC$Vk1{Y{7@Xi#v;4P!d)iUT{3={m>1nOdiG3 zg1q2IB*X_E+dwbGdce1B@&dllk3L3nGo?r?bB-4;czUOJ*icjbU^#}Sea_*m)*kVG zORBD2pBgzsA+8rGm!_-4AA>!$Zr9mIVeK#r#yhu*>4yBR>SS_!@H^JQu)_ttex zd&`m2Qm7Q!cEm8HS;f+mQ1a2Qib(OUf=;VO+^U0K*iLfeq05Bb2W&9Q)8`~>?~DjS zpljr0xir5eLC6={K(d^h;uwTVt522-%%Tt%KeckW)x6}b^IyPlwJS}j_lxWpuv&kqZsY|CU zyp43vHy%%?NaokKKXwC4@XY@#t0W35y;wFsY?{W8sasiPk?bnvLf628++QuuFMwt+ zE#SUeJnd7#SG ze26Zk!oKXn4X2C&*GS0W@tc46_ZYA-QLSroI@VQQ%_OUYB!{Oi3Vh%=!)RO5}FH$CD{PNk)qE|+qz+xS&Nq74b z`zeZB5brTtjrSk>vma)Sgei=QI)s{&Hc18PRf4O;Q?l$7tpnyBY4*oX+8lh>^g?C66O@Oakjq&9hMoH00jHM_c^o<@#`kG{acF3LvZA z1~FIP7q?_{{4`sJqCZ@K*%Yu<-@a33J$$V6CRpLWFW`zBP<+gTuD}O6F+ztJ`)-wp zP$LM`5@%3NDnpQPD**4(nCRW3RsJ#G&t$>v)UX?c>H`TG-Q~`GYzcl5yqW0u= zL=@iiN*MbciL2{%#L2Grg7Af@ilxCvII=>nuLrMn54btjx5@7?mZ0LUJ z{^6|*AnAF(yXC$wuK|V9*tWIFRA4B-z4reGmL+buFL*2$Ux6h)ernPpdEvzL?7iO` zbjOO~e7=G`PQl@e%c(JF#f@-V;D8GRb0c=II0T2O&e&1 zn}=`}xprhWc0EX3Z}LQx=WxQtY)}2wa0K-MqwQkW>nSR9!|@Gl?EMxcpi>|kXj(|N zyxtyLUdN5P=2E>V*#)M~B2x>B$+C9NA4ZEWEOy^y2;6Occ!Ezd%{m4PxXU^6 ze2Yj>9?|1dIFq;wlN0^A2`Bd!Cj`e7xVH#F2M+thRg6J#1VJjWyF$4IS3lsNq@I^O zl$(lxIv~`hQ|}&<-Q)s0Y%D_C&{GzPnd0EYp%mJga<6wq!v~iJ^C#a{RZY6AY$>ix z;KbOJfu{8x6eG9Ph{VsnT0C6y5Bw&<^>~=u0n;*aG5L_Ji*rw96U@C^I!Tu=>)4K8 z^H{5w5mB=VSR=odz)W`U@!2+cnfrd0RlAk96?RTP@Y5s(@O#vIA7M;WK)?#pf%2d( z4Gr9d$$&j6LbV+RgJs6NH9~MrQ(4Cso-So7W3^XW9AVnY^4C(J%?%yz+ImeA6{wwH z`?lk1FdBBqFK3vPmhHdOn~=7vkSt*{o*U?^lC!VbK2ct{=lu9}c~XGhsN0b7om~y& z+E>`Gs_jmNUN~#21z*!Ou$#5y!GekaFyq9&(5>Xai#-}T!9QTU9>SLw^lHD1qHDLo zg9Dt(hAJ;*L9zBjhQZyCF%?$*Gv0DNnM8K-xWcWBN<%cdOS7Q*m}`z21@0i#2YPt? z#}l#g#!cAtPn9(t<2}1YR9H}zW%kG|mJ;=tHpxwLjormz! zjcb$3+N1LVLlt*+<&Y)B4&MC4PkW{Ys~fr1V>K)ex7Qj!kPa4BB+Wju(9=sGb<6tL zUrfZRmj^7$R5quWN;_;%tA*beWkyrq43**F(uN!n&#-q{Xf~KRAZ2uM z4~%kyqpd~hK8VpZ0E#p>zG!R1P|sAs4(h_A`@WQd3hxwYBD9Z_JL#Opnr}5pIEBQF z$zf|GC^Ba8d*rNUBk?rQf|4t2bN`$X&ZZJ3LuqlJaQQUtx-5x;Rt%|0itmh3hcS>& zFh8Ra{Hj!gQ^b}#(WGWVttz4D7aJ7Z)8GNS z6VsqZ*`f`0@dX`6M(ffjAHE;n>tAnzA4xsM^Yd{v3g6tOM|ck09p+HxO}_{1T&i$Q z@%JHl=_}Z5l;O;RE$9xi{qGn$?MdAO;L_foUO3%o)lSgk?Y!KsH~O3WFmHr`{q}6v z2P^Nk&h^cYSXZsGZaZ^$lPT<_09s>cEgniv8bOm9aQ4ew+i#rSRs?-Cd&tdh9ZrH|YnK&+hM1XTkr zrtL$rs0(cw!~4E&*<>h-4W(kRVu~X?w&_`5`_xgb9dlqe8_);6!Vl+qp~BX^t(ffx z3Bs^QJ)Ny*bivLji8G%mmgj~v9{^jgDK^E^?Hu$DRH08ba^L$9YsE*Fh9=zoQVKmO$kI1&1rAA%al|M*K?>-^cz_?^EcNiN;?>yl zVh2bz__y;f{osONYm}tOg$rA*$uMXab?O?Bf;*thP|N4jnC^k;+{q2KF@o`QnFhB@ zwkk1O&a*Ihp_$~9N{|19ODItOIJSik&k#>^W+*v8*`pJNZ}@(EOC~3Skv~?!*Z|Xm zqpv> zSoW^6bn%qy9jus%0Q|ANGpM5L-Z{A5BLu1}X`cbutQq{0gi3F)V`BuoD4SHPCTl|Q zSnguKC>vot%=ZV}e!SOFR~i60bnQez}FL9mmGaxCgT zo|w?~j+Qf=wSa9cif}9e6kO)(_om3^iFA~q^2k7^m z%$cgVN6P}gpt5u5g^x|v3mwxe&&H`*pz3^BS*xDxpSjLnq`1tmPA0nC?CT%Hr7F+{ zng*xJRn>;T&~bQuUAC+{XikkmdA$#|7fS^C`agP6^64-eW2I#asEa5cO$WCPJ>7(} z?8f;ua%X})yTQE+n2y16UNO!ng!KA^FNkq-+Cn>~8=(Qv_(EJHz|}n76aE;uln+`m zaR1dGJIKiDD-HQj4OHI@;1jo?{)p&>-CRo&gY!_-1mLX=RWu6KKJtUD8_c|dRdpN`<1bG zG7_|!!M`@F{%BrL1Fyf945YDrpD_$V#x64R5FQ{V*!G>q!q|WH?6a;{pLlTWv9hBo z3q%^EjSm33OcT35hzgPjgarePEe^!9xYZtjd^kX%X{`G%`LHkaehCyX?AIyvOD^t9 z9wr#WuXFB~bo_7B@j@87kCQxxz#|GYJ|v1mg7&%buu<)3UeK)(I!fVQu<#EXL9LiG z-^!UGQq=p52*0TY)M-a^~kTItbP{S2;36j_~zZ22R)&Ka- zVUv(b<>I{mItuzIfTC@oD;JpI+b>Y5$iPweBk7EEsB(d5Yt$!PeWZ#0L4<*3$1bKnA8WB}33CcDv7|&el!pIwd8sy-eNm=n8sEwxqi7>#L6$2mZZ~Lp0 z3zC+(fC3kQ>tMOVFYdES3{+9xL{EEVoD9?>*3|OyR+Rl)-#24B)NCwFIZ?c9} zpCp+J!Xe!h`~zUn)zA_^?hJ)XilDM3$`Y`{m%@&0w0^y_I3QgKkS=@)SJ3PE@E1t1 ztN-Psgt(30o4QE)DM|ja;JvpqvUl|zp-f@?-YCX(#G-zFbWfJtlvRHsvQB6`3av%G zzqjHav1>CE99N&nuBj|DP3B-|kDw%4hwl0hN5}8@vmqmO_g4|B zgIkVqX242~?a7E*jqe%M;jiVPH4e0sHrZN(-u`Htq&cIpzC)a(*&|jt;BVjF4qs+M zM!JC%hzu~wVDXDL<508y*cW3Od$%=l%c$bi*N;eB43k&C!q1fMJz)2)@6`)_8Mp`? z6t6(7uIf+bum!w>j|ULP;xQvI1aS)fv(H<7Pjk=w*|ECqUZQt>7i9)e&Gy#}6^IsA z>mvwYc)WKHhQN24wNFl4ZQHKL~?DUt(90e!x62qk0_(pv^Cd8tHq|>{zYu)cNp`f}ZKEe&% z+mUSNYoP->R3?o$mMKA#c3vEQAq`a~xz~`F`dtFAFid!vXOKR&h~Kf?wLeGDpS-wD zwtw5B|CWPDzRjP@HQ%oVv++WZXf{j3<7vF3o04w|z$Yl5C9bFOAbDeHQ2KcZe2r4= zrz0cv>mwM%!?|SQg1E|Z7ir{#^wU=d_lg!@q4e9Loh|H^2Q@BygPKw)&z+dwv!C7? zso$1g?trp+FUlc7g`X#2-92lL z)>wr@OE8SfyGLLQK9rX^d)ld11=Q#MVXXy@EIOCGiO=(MKsPl*Iqo?)JKFSO_cj z``FoDH<$&H)5N9Z*x2tr&?9XHk!UKR(axpiC2S^H6 zpvUIk>mm&}!kuL#@go`>orB&lm1Zl}cU#n8#oO_9L$x2edA0K}uL!2Wg||pW1kIeH z9sBP_vrCrLbb!(I-`-bmV3xqw+~A}%)wP49+lkjBzg zX1lg+R}wqh>Y%FnsXOb+eQUXJB(+_y3M!Qxyc>u@iz%(2i+>qP$bxU1QXwNVfW!kY zqdm^eGsLPrerO{VKhEZm+L%tv((ZYLPubgIXyVKqKeEF?E6?QQKZh}I4IylKAxl%9 zgx27lf&RJpOYC&>juYlan#a8w#m>Yx_;Js;7GlHWcppa)7U%0V@v}vOJOss zzRWsMqOy}06^cJt0ladeIsYxS(>yIP2*L zXf4@&Iv=*zPG0@^9SDuv4cMJ5=%!TjUU)-NMeugL^FnGAo>0a%x2lg4e1qy^L#TLi zfj3c@vW=whMnP%cQdP#R=G5|+^GK{}6U1|0E@r;eA=`Uy$43!Bfn&WkeP}}gu6{L~ zFwuh&@JUxyH@MAKEa22_e_`A#ROhX|~sR$Hy$~cK`5j0~!k- z5Le}0q(Bj0QsZN}&SjhWlWEhv2n}->xo06e=L!a8h`_9RRb1fPVZ#Wh7qP0kd-(Cs zG>_{50~4r*P8mEfO|z7{iVUt;t93*o3VTRTJp9SlUpFkbbGJ3`)fGIjYYl>~@!4)i zu{Gm?sGw*LZT_I*j8o!dLui}2TSO&hEP>ees(_C5WVYwF+&V zz*lh0`v8+mDLxk*QRLnjs$u6rjS6CXz~%1I3RFclDc^7R;m)*zyEmx^Q&(maKq|wm z1QRgxPO88N25Qsif*aI{$72_|`YqdQ)UTbr8H6~cPAGf3$uJ~|FuL1|u^!}`)_|g3 zwApQTA3bItViTC!D4z73XnYEH6V>`14wHzjFimg&k!3S!a1R)v*qLC_ca8 zlE?GV-9r^S?|0W%A3|3J!!HFtf7lL=WQytn(u)}TgX?^Fs6^{7mB-gPW0A?F_!r|C zqOGIyjAO}md}xnYI;P*9Y5b4uo-Fz7o$)qQ{=1IF$tmZDj%VBXY=L5LLXE`@7#a_W zVzML=5|uxf8mAO!;xIfL>@Wr8|Dh(k$#R#0@CgOru};|;vzHd~j>8mwYnZb;;rP6B zz}ER&(e8WE^?nHy$@ryMOWxVMAmY2d!?)G$wx<{U0_3&?@s<=N_1uTM?$}Y2Hps{H zbNf;F;K;A}G#u17Y7MebDeV)Rdp^PU_dLai+Q%4wHwwUo_BGj{W8vt>zXXpb&=-Co z^#4^yH*uu%k2kdY@l{}HiW7}zCJ*zHM;B^$St%h>ly95C_ME{Mbs^-NzU4o+joM>} zKDTm{I-m1Cvzw*k9F&AJl+BfUUdl1|6YgkU7hW6%3WSAIK9lg4u+3P@^-UF#6P))+ z{{Y+ZP9n4NL-4Z312hBm{*sDK&)3iDBok3!k$<(g7$>A1<=R*07_LT84V&#lgiDL; zSsEZ-0VmN6gk*smag=(F4#sd*cG%Du;cfpxWSdyhOH1#fEC71ec02Jp;t&#S%!r$# zA7-A#^Ul-#fbcN1^BMQNgpq3zVzoH3%P6vl!0_D%1@oMrpq3U)P|0{|Hodn)cHD9& zPh)m(ixYSQguBo;iKSFxfGrks)o5q3O@59t7jxx98jx1e<;MARB8K$%Hs&doI!U(i z1O>zw0Ac|_tRRoS_wBjUm&0RBXC01i8hRvzFO1+!3%f}f2V7zKJ}_Weizsmg;X(iM zCj5vqnRl_TQ3i11Y=Vk&H-WrLuA{j(5s|S)L53xXhysPl5FUU#1eHoIO4^i-Ml>#%J2mFs>r&A0}G7wbcwIg&~1b-;3!xm!-*eU zaXDYX;WCxdA`rlfG*~aLcC|gU=eeXGi{T~UzJ&)=)T1;3iPzIGW*{na zlBYdHSAbG`Znfem#n6$HV2EjziHUIKW+)p{l^oQuF$e77GBvJJX$ z5tuk>sv10scmX_HZ*2fPdQ(kqi~=c;6C~jFK@A=H+jCFw-h8_F;C%Bx;5qgc;zCP4 zm~Nl4#mEgF1XR6N$iJC6G}?FZ%z$u$jfU(q-dc*3W!)jb5VhC=1@FE1&}o@9+s8x&6n+STHGHH@npynC?G zsnX*zg}c|a-x+0gznC+k5{&$_&vRe5#B?9F#Z2(-OtC#RUFtQa4>*c%%=M(O`Wchs z^P{otU5X4_#uPR)T#PPvcu@B!$ffTsWg+|TB0cFWMB7Tk?V26wtka9h_0nEY%rm+(2le$MY3q$+ff()pLf${rN0X7@jPM>84 z8ayt8P_SjR-S9QLs8k;R{V`tP%Yyj(S|zvvaD_lodt{l!0R7Bnp;)(TJ& zDRQ6Z15*RnFCH+;+~EwC_{A`G*9j4VHTZl3{10dAb z43ykc`x9#CwDF9!1Pn!IgJEU?{Nk!jE+-PZ zZ@a-a#&jSTf1$tiLD!4?ny;oWc+JAk`R%&P4cSbMDo45n0eUIG0e-g;lQAztoLo(? zI1L0Mz@fg$Wbc(8f*-ww2!uckMDpH!_4ZKr>GEhCa+4*W8MUfpb)7gtuwL|udr@V{ zE}q%F{JlKFpPh!gsfN1I@;5egVI`WJ`+PkBu?YXoRY(!_Z=@i`bWeYA6q0e^H(?NI zuHST1xrqN>tpBF~Zmh=Gd(}&dpaZ)^Z|TSVNk8LC$Ea-8zzd74Db;PriXLB^h&dz<*?k^DW;Bb0)Yc$Y-)m^scGAr+I z?uY3gpWgE%8vdi7VkZeM0O!_ZaiGN|GY5~%{;`9Ep> zHw-Xru89azj|X)u4taw8p>^DFZ@&Q~13T2=4H{GpclR4uKtl|r!PszTzXABi9x|jf zIQ(jWHkLO0KG=Q@x2@_TJ^7x8o*4$GZfVrx*eh!>!~JfCyYid{ZVKQ%05CZ8k-V$J z&zeBSzY`qe_t_)AC9QbV!7vZ^!Rr1677%gX1@O<{9#~UXqfaQV_K-)7nmEqH74Xiq zq}@kIBoKMR!_lHA{s0RcEy+Z-7Qo9;(;!WBKV`AIvfBe3Kyg%BdRSiceF|0Y0afAX zHbx3J{vwGQ8Gi_zp?^mODkPmi{5v8znNQOpf`6eh(7@&JJXk=;ahw}~E<6iBiwQ5- zH=XQcko)ngqcy-eN$2@~M``pT1uZb{9S#Exrj@PRcIiUq;G&sz(V|^!PEriGak_za^MCtgP(s3PD z#KICmCnBtM%#a;)0u^;=l3QFPQ)cE&&e``f6SNvEJL#>(+an&(Rast6AxovLaRrW| z4Z@vIr+y~C7vFZ3QLt#|=Jt)-&GixZJCsJ3>8SR;ZfsiAse!>%IzipD3Kn8(<( zWAYo*%g;3SmXB?6dc3f_yn5ttodMJ`bwF1`Mr7v-&3OhGoV8u;tqT`&N~$FyDJXYu zrCM_&hr4>x&KddQ09i0|`dt=tfFz#lJ6Ad^FWo!4V*cO(quisu-(&IZ62dA4Wf@kv zeZ+y+Ze7*cv3|uH#SXRA)w&W#u=c0j5NAxhCp+q8RXybecfb3t``x>$;*nnOLSQI= z4nMWP>C-^|Abfr=riz15@4Ldr3v}RGa6gbwS1R1O;Vo5)ZRkFCg|mk_Us4EacDX^* z@oQ(2(uu0730<5*AE!m!%B+?KZC?KSiZ%%)4God|Pr$%&15Cfw$!SC_q4DWzXcG2% z{+`zJsnZhk3v7WsPFH12Bbb)QB$)`KrLy*KLeO&80Z8D8NHF3RE$cJ2~ zE0UMg5@H=spRSJcRdalmy8v_gcwNhqS=`=qV7}b^jDI+v&GRYk#7=RpIK8>M0fn|F zHmk@-=8D7ou&xNfjq~e%VF`|IcBU`Pzqau@iYC>{UkS}mDJYW&^hrzYgjICXoOidI z4g2ICj!6)iZNHH^?HwP%U6nf!2?G%XS539_USml`FDucW{XZ5Oy3E%O=TVm#Z1XSU zA1kMPN%YMno09XGOFFfy-SgvUT)06ku5d$#%DY|l6EAqKwd?QkJj<18UXx0{&{Zfm z+)=FB8|yL3aXT+(mBF&~wX>Oc>h;G?&Nj5nS1JY{tXuZON-SrLzZh%cP7}57me>xo z;ycv1Uta%~Lb_#QHeIwEy`9M18KOrjx4UM;y}rRgDze4KpH5SHHx(jpCN=m;97?^7 zznJ-1;=)FpF7%Ssz1aELZq+u}6{Sxm3p^4xl)aiX{ALJdhCO4O?OP(NIQ<4;6?!?O zc}(a5_cua($ESa6n|&R9XH|6yakq0^Q>Ja%lO3d@YZR8UwKGx3ErqR5o4rJ(H>L?b+3u-wZAj+@v^v8>iX_|zt%Gq73@tK+^~SLCL1iZzqw9>T52mzwYq!4e~gDF#>bC)$EF@F4{(ACd`P*G|4Goc zFd_)$9lPVp1dCMPn$tdjMmUR?KO%~`9>CKdnXBoiUSdz_E~pS97?o7F*S4-T@mtv+ zb}!b7&y9}Cm+9uOvynU)TJ<_e@p;OU(buC8Se~qO4D(l9u8IY4yp-Uysy0F&p7Bf_ zKmL>h-eN>j!=I4cSh6p>C-naQJW}G1GlVF{J!Itv%`@d2=f#5j2X1HbDWXjJa%L)Y zznH8QRGV)-Ka^1WM7Ik5dM$M1^(iEGaAx02&-|vTtoayj%W&qsWnNGOkuO(MP?mmp zJNR<7Y_MhD&7-BxdbjWsVmx}JIQ`cwVq!*O??Gy#J-*xQsWb@o!}KOsll&bzS69r+ zWhc+v?+-&4vlvG=Da*p;v(%K~EnbydShUe}-P)-ol%BV~RI<`+36rGh4OJV5FG{|9 z3JJc&^w`waIsYGn%yLn#>iQDcl;LmKkf@)lrsF4^hkCS+n!YnmMOUyUso>+SmOg!E zHfew#7|}$YNsq3}8@g->>%%|ExUqBtRI?1ZKJ~DkNudGFEZ3w{?3!t!ZJ^)h*mLg1 zX)ZmKW}X`AlUOZ~y`pm4nYFP`GrOE5PkYwPFb77uZyF{3XsH{iw_QE%yxM2}lgs?n zDJ|}l*&w+<4_KV4)1|I}S~N$WLVM@cjZSiw>`gmo95UaqAb~<}w_np!HTXuM;@zzc zpFr4!Zud97lPO~^R^RfJa9$uu=<&IdS4CXc>g^eVcyKc{65L8XI(gQ< z=vBSFWtqi$F=hE~V*F1Vzbht!Xf=tK@aGYJs9Xo9?)f6)Wi7k;8+YT;9p&o!uCD?L0Jm|3URr$(0z*B}8^} znzL8Kgg;kxI7Ze973sm5SWV|X=Q+~e)eJ%Szhzss{BcxyCrFevWqZMHLuB0g-Ui}| zjbLkCXXom2>u$9|0sPu$-ZSasGlM4j{Ij_uVg_doZHnCwth;{ei;V#anSOIL+QTG} z^k&db5-(Kz1X|go_$zw; zzCT5EFZGNMi=9_@!+LDQORDusqWmuJ_D!RPUDZq3bmSYIo(O0~0$n$j329UVRZ)pMxqBK96 zry9H;41zDa%7jGqLmdN8q7Iwdl;cNj_r!Zko?ETXdsM);!yC%9TT~XTp~OH9A8I*0 zqDNQ1Tk!WRhy1x`*juD5*JyM-MnS^T>)kg|yeiiu)3wm?Jh7*D0zGD>e&z?B#P1qU zz@N;grOf&U&U;yt3)cJshIPy|1k*Q$MqR5uJUf4_>&E15LsDoeX&IJZ!M>+uWf~yk zU8%zT`gjObX`ANR+n0z!ASII2S*ABDwFnp-SNbS3ePKp!BF+{LKEbL+Imf;po1 zV$6G3Q){2j_M*Ihj)A-y61tD2(k#CC;HNKfMy1hsD4h0*u4}lHZ@UyCs2ZOZPZKBC z+0ixLXQbSWlpo|Zb!0J~KagEv!*awNOJ(l=YO9)G-1&g@rR(24Y-07u*>%s*+<{yN zxzXzZek|>z$`#@tZ}i#l)960FDbnN5*3$z*8wBj?qL0%(@t|E64MkqP%3EFE0+8zg zZo=$Dn}TseU+ zSGFzZG%voJzJG3dE(L9~QTTa6zB%@3CpO5wZ+Vzz*iYWkve@$UR`17WFY2#uJ$H6; z7SzL`OT!O)Z%3fUA1TnqR+x*^cdErYCQ;t8wlK6wTyjVs$Xa%o2<3U52d+~AvNnyW`o;==LQSU_|{KD+UT1hTJ7a=UeU?+S>xAM!ikZrA)N zR1N;h>1RD0@Q(%XufMzjZi4d_clJ$LL|kADjPzB#8b{ozZZi?f^z}_{kioUc%e?1e zMJDra5_YOj&HBzSP6wQUtlt&Ycs@bc?oe^beiY$}8ZOrLC%ys7a zj^~q$<2LQpAT8XogQ@Enmc~l}E}~h>X9|XJ&TTD%U2Ewh8&n12E^wRoF>*V0_YwxsIF`a`1|K z)v-O~s}i-jfiEsSF$E!gAxJ3qJvk3*V2kn=+^1@l$)W z9bsq>wlDVoktbv(5=p@*|Z@Se|1Ivl%6A`xTV8sgoyS%2fI-7 z?)6(gS~;8_T#h&89_qh)a`|FX1$I4(DvB^QDW^I)R?2q<|GCX3&F{A04#&&nb~9_T&sopUQ7(_!S|0EiYx=AhxGA0F9#I1BfK@I%{G5~hNX}VS zKF+qCORmZ%(xJWl>W0iu7^cluH5ySRFX@m`b;dw|&Ft~3N0ki9#f`7c3(9rV7AA`la>bo&h zZQE9o`0rD}@Y&|sOF8Pj5`$yUOpYrp<#+}};C5a!7usv~J-N)(9s(|eMS(%f~GS8#^woFsdm9d`nkx=8rSU9kI(+@|kB>Q!)zFXkv@M6Fxg6`7Hj4!@>3t zmcNg;Nw0dUCe+KX1T$xx_sipO#ckMq>`YA}3Wfxn_St$nv{%LX_KS=R9Q*p2y;|Re zOsKNnLmImY8M|&D3*C&rcnXmsdT5sHYY0sFoDguhd+<>Pi1%0{)hD} zRKhoBUr}0ge1=BJDiz=e-Ao%6^+elhJi9EWg-4CNDu-iO;<5 zEY7xy?+cVL41%h3>AW2B9l?jHkDtLO&#y8$PAne!NGDHkSpbr#QH*4i{UCuaNYT*v zoXj_!r!+t?ysRFyOZ%NDJtivOwY}+dSN#+`=eg2|N{r{$BcAOfd0_I|e_=it7^70J zPlujA{=!RP^Bo)W8o|d~8++7!xw-9BY59P{n$^cdQ&Zf-p{j{fT=%f2xL?io+OPgI z%EOih7pk(Ag;ZwIq+B(hjc+nrJ4byM<#UGpbo+cYdge7Gp5#1z(OSGSEk#!H&Gr@# z%q|)*sO`&LbYFV>A6}KMPghiT0(3DWcCYph`wwr^ky(b{$X}glF?^?9uV;$0Q`fS} zR@2d->pkFOoaS>2OL_G{!J5BbnAPFqK0k-V)HVVL{h4oyJ%CD+lN75S?1%d%*gN=g z0oj+EBb1q!tKz$Ms-j|?ABDhZ0 zwC2<40bAe+vS2bYWXJr3NC6{(ui+RCt%SB$OshBMWjNZ=?OgJ9$SiOLTHOW1v_3^{ zLdm&$p*FbW-kQ9h;piOQ_q&*y@5<)|!A*be9cL|I-tCe_-K?bpWz%sYd4!PBo~yUi zgSyC~-+_o~V@$i~MI1Bv;e`S*jJ2JM2tvwj$N1Lbz2;k=Lg9B3fKy`f*WGN&PB3D-#@hsERt_F{!>fdyaJIW`0NqJ zBherQ_t{UD{dJFB)(2L-hWQxV_NrJO^WI04n9MTwg*jr0z; zYqt?Qs~J5LA04G2$VhM|3Ne0_97xG~Pnv6+8m#WjtE`u!2Cx~h{92W#+7h?@9_`@X zVDA>9`G8AJK1i;`{(6zTg&OE^edtQ#e4Wt-2Ll0>_d((-t;*`}zU0!SxKEEm4k@{z z9se9~x`TYT{e4;6)VYn(oWB}p(tJ@<;x+EFicwHy=d^kxw5uAk064-#}>03p%C+=KGbzT5yaV(=%aE1Ys=hmC`(k7t{l? zmd^O7e(E!$o&_2-i}jJnLt@x8lka;RZFCG98&pV>qo+ldA`f{^?O%GmCrv;3T$~td zBnnK5O)$Rf$H%BrlA8pS?|>|Koi0dLEWdhQyUUKm9q3=b{k(J@?JUt=axfD7HK<)J zc=7{uB8hVjTvf^1mV;OkljC?AhtpD^+lmlt4l(V%lN?!T^iyfO4ZW5%;DVTxg-!(lR71ynu|q8Q|A z#Ce)Y$d(12cv1enNaa?${+hr0kOA{4&#IAtz)hh08b?S<9G}7 z`cd1HPaf05OrRfTw3V|@u!WbZ7w7?LC(uct8v6FbEdREGcCCcQWO_=g5gz-9R$UX{0m&bEhpo=N@ z9)CeK@a6)#4Fr%tD!VOV8nbM1xz7ZL2PePX`da&~+Y||XKtWTyD2ZryR{27|%PaZa zOyA%WgQsA51V_p+p3m~IbXoivy1Vz4SA%gSib@P%YlJx#B@&TJ%32)@ANz8;u9@`l zWvQI)8u8UohTBROA=n0?4Y{>&GdxmEip&fKG_v0j75Vq?w}0<${x3q`C6B&|6hwOrG4_B}Gb@>Mp`Ph9wBa?afdQI|g99FRzlBdg(1+na`I&!oDl}H5@eTMl zv=ckd+t`1hY%{jpKdWa;FUr8EGw=XS-+Vc{H+A*_t54>~d=#y&rXP4zw<*O)EPRRZ>)7|j*Ni%s3WfaQm3<~A{2!*1; zTmIuH6y^d7MY@ebiM>Igm>u7hs7t^Hd+sR8-9T+3|HYOizlL}AIm+L1L7|RDA^+2$ zF%LuGOhB$#|Nj2wz{!6fseFXm_wNU1cFF(y`-M|7 zJO25@e;>`Ee||?P{hujN|C5A5o2BoE!$=_;@<$Zd_wb;fKYy-aWtBqUihI(UzjRqZ zK=tk0x449+csYZ0kF8^)KZ=}RQPOBbw86;iv%k-#>1G>8^q6RAg`QE4)Ec3THsVb< zmik@AgdKV%&>Dt@-R1#*@kc1CBx`0G6zU9B=dU(qfBGb`LZ$MYJlUVy$j8S=Ae#7P z`un5cy*8Vk{1KbJuE_AWZ>?L0j-3>ZRw*{*SCqOgTwjZ)Mb;ognKikGLiI-~C@7@O zE%$$Zm|Z08K!Y2bjih^-)rR9z41M`B_UqTL`t;{uW7WZ|>2Kb=;q^MQx;WL6XWep` znYqU5js77fro^J6BDz&}-(^$L6T$kMe;Msh*Ui@P@$o4uVn$|4X+#*bQg~#l?l*RI zk^9uCtGL!ro-T?c=VC+c;52SNzOV$Fke4YL70t~WL&oVE`ubis<1fOjkwVskly0L^ zHzR^e`zOzNbD}0;JzCmtoho*G}L%{X@%=}tPN`c7t+S~QY z))%{E`*Yxn^hYmLT1mT2>QSb;h#phU#Xzhx->R!^D*b7Yf#>Gt_&uc)6BCg??wFYg zd;Go)&&Y&G%sstq*Lk;O?AAvMG!J{&NV(qs3J?BSL z9s0^^>*`>=QGsICc$I5OaSf%Y$MdZJD@$KtZtDkQ!B72(sWW*oQ zw8;@7aYw9i{>`eV0<+ZE^jHayVMuU##nY)`33e~L`glJx3!ZkVjOp;&|d10a@J)>p9HSz%s`gur* zu8+?ri=S`QM^>Ul9DrLtIT@i9l$Et%&PWjl-ydoXl8G@|Y=fdF7{1likk^;`vfHdu z5tfa-Y!^9E=5*zR^OX~|KYldvW`EL{DE3~Qyx6wG8Bg}D_@s;tfBiZhxQ9jasMraH z`bgorg`81}ks-h2=Gr_i0VLWscLS{* zFL;6>fX#2sDKRc`sgUvNhq=%@Qi-#9uUfg2Um2 z%&PxFk7&Avg@@ZLj5j55uO0gP@35E{ezez2*%D_MEr*knQ)hu))InE|J%9C{M-I(> zol4QK@GXPWVuFD;*T$rA2@4CZz&uBna4|7ipdxt%1RD45dLJL3??thJRs8gj0%41b zi<6DkSP^R`}_0A#Yn6Szpj?#Hz-g#E@Cef z>}g+`n3Mz>Ql_iGPEgWi$=g6*-{#TLrluyKOIv|m7r$s^P5p@TF?Aho&X zl0C+C=1ioN?X>@eix<`5+p<$(LEj^#{d^PgryROxDthdRx#o3wc3qc;+9QQ+InmOa zD~&$wd&I=W%U zD?(liRxORMSB>>(x9mRY3<3<)Y++l0zrMM+qV00JrdTAs@;I%3#a^u*f8A5+u83Ql z>2t`77IjJT!D5+MSkh?OeLh?>X-QVq_3|odZfOz5Qe9hw?t%nRspRbR4D+563@+j6 z)4exu-uwYFkTulN_pO^F0@jds_s5};(NUYxpKoD<<%s1O!SoEM!9d2IBTS%edtKKQ zh@G=ZCKbNbi!TJqQzX3?Q~WkJWTUlr)1uL61yxmDIEd36925%C}P& z$+n+raIVAO54LP7HV^-J&2-?vNfQ$j<#$(d^p^%j#VosvoSem?!CqdHx^8W4&D7wz zHd_Px(Ko%(4)TiZEXS;dWi`#KBE3$K^Z3O(kMs-e`I1{w;k1U{DqN~MHBlv|%O6p9 z^w<^Cs$IhN-R%pviiU#4Inf|>f)7+09c%4cQ`AX>^avmoo>gN!yv- zxub;N-H76lkmzo=hhxmAx`#|UNsqlKzAmrYDYC-$U}dx5JG%Y*)0pg@Zx-7fSoq8xFI0s5~28 zxmgx9cJA;*RSM(MIouIG{i8L&?Us)WS5{V-sLlZ*8lJHSjvY&lmkTj=ri!9%niHPo z*cEn_1v}F1+n2hqRvmPlOwctUc+Zq21G-`Ptb}Q6amwx)_M8HawgvrdjBg@Dieb|1 z%U3Mc$D}teW~vI=bqd*a6=XJ_bOMDX>mg*>aN_af#}g}K_-rAywDUsqmwUQ8K3WQ_ zN=d9Pz0=6z0ftD=Wp}8_mYpB151M-O<|sXzL{W1Ahg`6b6j<~xdsV00N-ti@fMI1< zj|p15F)r11d#+Kn=Ns~#`QEheUa@ZiDRj(a*0KtB=7^=0%$*ewFys*}?pMjr6#Qq-3sUgvk$hwiV zw(;{3b&1~7#d5FDC~Ls5h)Drc1=_9_F>!Q_0&|i?d#~AV%!m+yz-oDB0x^$w#hkjS zq7po7>>1UdgnRzt1tD3Lot{NVetxWx*yT`R_ek5I&|bKwG|KXd2#|D$EnS@D$LrO1 zDJkbPH>VqKTUc1+Q)mA&eE_9y_mj=M`|;7uW!=NW;b zcXEg^wb?i=+v*D2b+(e4!SD31sHn)9in(_F%$aK2E(Al|e_p80Vej7U z6wqYfmT~KOaBv$uly~PFon~dfU8v^OtM>tCr>ANr+lm}y=I4Vdzr0>moPY3Ohh6WdTXG?6!pd(KN245UV8Emg_u#omP`03Z?79QM+yWjG zzSn-CY=iB!ni~+a+^ePUI$0zKMH`375%(rZ0YgtE+a9I0p z%;U)+erxeL0^;IbZ$^KzUdYhPx%QBjQAb9`pSmz%mKQB-8-7H<_#TVVFgNVcLlrE6 z()~ww88AE_VNaNe=9{){85gHO;Jp(y*F1-gX@PN-KXvMqn9FD&h@V#GykxS)ix)3i z{#KQU1Mm#S4Y=6~Mki#AIeR>D1KZe&C>na>GfHn_mzO;INLgM(QEN zAW|PA8D%<&LMeKvxL{H#p8V#B!K3owlx)dG-@bkGq&ETJ01(4o|4TKiOtG!Y(o~E^ zeb4BiZBd0!M6DbLm^9Hm9v@zb{2mUlG)z%em1Cj-vs78WKf9Lw#R z0`*5SCpe4lp=Z$=_^5H~yO_n#qh*^LtH5ncF)RB8KHa8sd_sTR!mIJlojW{%NnQ(0 za+J0CM&0EMn8c`Q?-KZiL)hPIp0_yzsMA=|d|;IEqbfkYrSyOuZm`-H0vU?(x>;isT?T9Z}r3ST%1 z(XY>>Dn{^AVUzh7vJS|X9KRHiNYoYh^!q_`b8}$RIk!?Hz`zLRmM!&q*Z%w|RHmvJe*U8D$KAVj9mS`=ke2S1QU%@I+@52W z4+sl3G=%jYeQT*SV`mN?XeQ9nrY4w`+UU0d*cF9%fLQ9n#cQQ6Uml1*)7ThSD2Byi zdGja|eJyZ4K(i_yqA;#jPQQ5PFM#C7@uhG&x_ceKW&!gB0WLk&9((m+p3{)*(0X{S z9L?fHOHE40yp#mq)O>ZOc#I7YNb6AoCO_^Hs{a9)Xj||KWuJzIg(ZU1r*CuwPF-ca ze|4r}vzK(*(eBgls{LY){r-q)87lOnL0LWRwmnfS_Y0s?tL5|zIGRUe0WGwXoWZ@v zq`e~S?(Y$tidrUG|KKnlVPS3D;lqa$kif|A{?9DnaGK zG9X&|&6^Zma5QpPKXy&Y`T^%?U%U2z(;6JM{6pVE!MmJn&jFLsGTs!Qoe}~8DO+R9 z>o@+5q{^_499uLzk*kE4g0O=kXiiOV;ln{ZCt8wqJ4}p?W0F*&^{xG;hrj@UmiofA zTB>j2w@%SsI>^G}rEGgItq?Gf zaEePY{M_ZsI-rJh0D4^viw2_2jnsjSyBBCo>MG1;^BO;eC<5#!2n&IB*iwfsV=CZP zB2)!z4hrub0H-Dd(c==d({+#yM_zGy??oYTp^;dP+hltj_{K>50fI-v-(Nikr&2k4 z%y{#=p8}1CMLbq>W>&Lo3I1`e^!8fk-<5w z;TO8Sdw+p?&fIuo`1gO}%dJ-dylo?w=I^gKcBqYDcQii=~;#evqp(@2f>g}UR7 zto=pcuAc*Y3T`>*M*U*v#EXZ{2?}ZgdBK^37)n@=eMFDFE*#^bLx=M2bzI`$ObZ0m z1Q0LK_U4TnN5>l93L?PuQm65!JsRrjuYeB@A2<*T>U4-YJ+Z;h(8%UJd6DDF^JmXq zLBOf=vb1#H(YH+CGPUZM9Q-#}u0~s&>`n%HTYvo-R$E_RHn;)m{VW>zjQ~1+giKBc z6Keef$E6@I{|iEzJQ;68qoeKUULgDdxCgeJ3XxX`Gvf5~tm5Fy6Q60>g&ssAfk>~I zu|}|nfX&*TGw~|XG=GC|1}~ba3W3vBoq5)PCWNvOY{wx1<6n@~DBk(;=8GfNa8^lU zu-I$_2gOJf8>`}p-%0-l?`3Xog@Zv1TIKyp-KKDy0D+9!JR!CVB*g{(mgm7s&49JS zxtbn+nG6ocTUiz){D{kQo9g7rCW2RhAWvla=ovDZ7owN|T*2=%%AFn`w`y5=_39Na zp|w>DLcZvcmS5;*1RX>^fdk|PP60FkYFO@Zxn;w<`J31x0a?4G}f)nX7Z^=wGEl)GMb~742TExAUKrjKA1-02q6fbh3?&ti`wE5a30& zzp-zK0*aW?#{r1a?F6mbXzK^32@J-RHRUGE5iEf~A&r&qNe}~ko24B+F~c9Qx>7=n zN_*~oL+|?4|59keE~84Y`3ydzlDupFLylJ#et%(9#6f_@z~nLx~vL4F`Iif|Md00^gDmL2|DU9Dql#L>8Mc55 z&8hi|-GgW)sjs3UT)-r2c=#^I_&#v9At2M5Z6b}BG#x*6b#(;NLl`Y+19Y@(;L(A6 zX%I35vV%f^NP%8FI6`EIOn(}rJOrvM)7%2}4N)8ch*&Ctds_l7PtdP>z)Fa=g|KEA zI3vKvybLe95v&_@Tm*4OWo5?7v%xnz8lS`Rp512?%%XwOuB;$#%_Mo;5q`rI0O_|N z5QDr@xO^bfNr|c(X@M zFVQxXgLT>p^%r$-zyb_;2VNzx?EgU`QAoM6I$L>~ty%m_9{)=#)c^Lhn)N*-cWP)0 z3K5isE?@>iBC@P}9uAJ{8yg!s5i}^7hs=OU0I=89)YOnP(t=>UY7Swcp;q6XGa>kL z`;l;u`FfE|ei+W0{u>ekVbcy>fMr(N{XS<=b(>PmrxA4TF9;){WWMxo;CSG@{J-wd zL%cV`ZD}V**r#cZMC=>xREZm2UIpJp)xQqmI=zLJ`GvO5qa7z=k8c0{>mh_X4+c;x6rnlt#!q>LHoTC zyCu)i>oTf`VoA?$#WGZ@}_cBc1y$Q{Jp=4nO~^273MCi%L^~1dU07Hg$d~}fbV8!w)w}7 zG=k;bnXEa7rnM?z75qgCr8L4KJvBRoP`IsW{K;GCY2s5$4CGxHW^z#Dsw9tP7nJH+=6O3SZY;T{_WfZKmh$!%dd?m3BAf7oBUkae57HVg=76Tfdr(sH3&{4geoc9B%-cz(WL#w+sf zJ%yp1*tf53C&i1T<=7q2x=j;;oe{6N&?LzZ0SR-C)`Ysvmz;1Mx?RC>@IfNbMxm#m zMkv#`DfLc0ejZo;^M`P!?+mi;i>I|Qkqs|6n@kGdc*P^8oq_QD zen!FksYA!&d+N&f|tYa*z%k zJB6Nem`E<{^-H|APO#f5i$M8DB2cUnGn%s+Ggne^)4#&vXK^37LKhHIlCv{)*~5Lg#giiL zTEIuu3$DFIL^)>{Rd?n?Z6((X@JZ64oMv?8uT7W~N00O{@Sn|I-;&J!zUIbuDbP$H%3%e3KvSGcpH4@0%ak;W z+EhW~*!(=&@V>2V1US6z5cxXh423V__5tgC|_j|)oOr(1x;6YtV*d=}ZGfRkD| zIh0GbW@bxzGkcw~{IP`#4ckp|$-?p4|2fk1?Ok577?ekpV~s&G#m|XXOGn2-%=dJK zjNcjs5fW>Ve_#qHbLNsASo5Z4)$lm_LNT%;oXge zF5QJf-9Gi#?3csWM;VY0TN>WenYNFu=2C_iR;oMY*~{=r-f|KerkM$g2klxG+#m#IFNUwUTea@}=% zHh)Xz8+Sbn*$B}oTVs@T>(ea3=$Qajn)FOkfLZtaHD#CQ*FWIBFgK%1>eyWWc-q@t zo5*%hL8aT`#K(?%Or41CXm4l#G23Bkr@wSxf7f!l96k$|Ky%%j?2ug^54@uh+{jSp z>mrBNxKrgVHg&!^z9O0@s$OTxZR-S4DYuZof#G$qM0v{&6)jzLo2jZ&-+zJZqep1! zLB|N+wqoyan7)nPdq~hXh10sHPh?_W921|iH?unzRi7ly7nLqzY#gslmU`KS zJ(=qBykwtd_0a+Ce>u$JzrvVGZ!g`%5)4*B)f>e;Wqo>I?U=-6auW!y6WICoR<*5ejzd+HV|I2eA$^*O`~v z522gm8zo}HuL3XyQztR0D4jy_Qx4}uqPdLHYaa+)p5OL@9=3T^Z;|dY@JfqqkG4o? zs(oo)F_E-OH=^Ya3+N_8@N}EBh=ADtPnPZ|{QU?ey@H&ml_yBMXuGMK$HV@OQ-O!U8M5>Qm{8h^(h^t@KRkMEKi~ zdP`qdcZ(}YFIeBfaHCN2r@{Y^)ejy$LF^ofT2P+7wf%q^<5c671dae_;eog&o8!K( zkwCBvqM{Og44ivyJz7gm@(78@*v94wi3+$9IZ9|qNC;shmTr1U0awl~m7ZTGVm;cj zBzR~a>b59iY4}Zt3=EV8nj~LO6YL5Z!|E+QmI%QJmAvlfoEkT~(_{N@r`KI&n)dsB z3cD~s>oqo22*C{b<9(g)XBjuXZ>1d#Igld1SpL|EbNhLCE;LRRO@x-&BE$$n|u%9v2;M? zpsC*V_s2YU!58`?KY$U&=*)*ie*pL{vr`U8u=Q%CVg14pIAtcff^gjS_{azTVzC|c zB6>(5eb0XY`v3M&c)}LQg(JhdBo*nI_=ROxRqaSuydO|mc{0EJ;*I+CwTx|Mn%|DO z#y$xR4LxT{IbJKTm|t>OE6%;b>RYe$O(`iUB!K|5+R`*o5VOn?`7 z=uu#v7AsXkQF?Xd3tgdmZV7^iV^t8o0&1jdX=n@zOVCYs){vU}2ky4Zi)~Owa~FT7M5?wBMTM$V^ApyHH5FTu*pJ zaImVqE?v0AIk64ae~Nq^CB-!j@9EE<^I5*p)9Srk}fEc(ElJa{4a&f z|0Eqq4{~!aMLUZLtbP*w_KlgkTySGdg;@{FOL-DvqPQOWs-AQSHs``8FFWE;_d zmyDnOo6S@o0j;q|wpqo>QHuI|r=~~i`72t{HA?CqV-Hk46B}HesdX(lX+kSf9{2I= zJSRH2V$E>HyMUb3dp(1n5uQ(3+O9nXc22_K4wcO>8dQ^yOqHY%Rq?I#nN@p3NH5ui zawneXn35S)@$awg!m3%6-MNYN4bqn(@#Hf~mCCUCjt_ zcA+V8cWMH`9ARn-+m<(B7eK`WacTY;OyPEv<6~En>8El^xk+tuaRukZ z@v(ifr;rT``~Mlxh+85&e(k!q@+l2`X|f)Y^@KRcp91_W@n;o&N(a{i(y@*A>Bzmh zVs|-DX2n(X>!*$KmGG0BX17O*0FD zP|o7#@2Fy9{@dDvvO}{S(xD2`-26JSUTF+tcX>JZ2qb5JeIgh|u_tmV?tNCmOYu$M}&N4g~shxLUuF z(Z2A_7#QtQOJ1}4b_wgW{9*d7Yc?UVa;KlMkqLvn&-HILA)s5orjwGF@0RyQYR+0Z z-JayIx%=@JHVW1%Q?^Asr8(yi?d3~u-0EVpFqic;`mCF&)uaGOcXZ z7hup!ti1Y=iKVTYGb z&P|TBW{BEW#170d{-tA8qe5!s)py|c(YJi@f)ky+Hga=)nsNVKv(Sf${2TLWXhG?o zxqb)RdqJ$Nfk}K1hS`tA^=FmkdlP0b8~1WJtv4eh2s6x1)T{BltD;oXkE5hZ&Yw=p z1s76}Y*93Q3qSgo428C&4!%D1qB0Z9UK7R<*R?^GtD*`p!@MimQ4?=Y>fpo}U?AyHM;EOK-nf$bD z`g!Dgn0+Z{8kZpXb}+pTkruVxTUtAoMa+$CeGf0Hc@HC6iQ&F*y18A-Z!vIN9i{B} z{~KYn>Cp{d7c&{mErEg~6e{db;;FS{@bwDSapYP$a)c=f!*+rEty)f0w~OxOT%Dw5e|>ky zrN;*-_ncmBtXK0!&P7RIIcks+4iy_2nH&azvPOGLN;InQL@N^Dwc`~YByd3Ha;D`ywX?|4p1vG?=s<1E^X zs4_=Z1Up3**n`xPM6GrW9AwFmy&UDmz!G30)k_@=)Y66p3P>zSI~ z&*@nSw~}R}*(7|o5X{`^c6v!MU2%uws{`KdJyMtgf;lpD%C0R|BPB+OwFgZN$wAC} zXaDK?hu60b;1co`tp{7Sgrd=6u?@N5JtE&W2O*#Tudw@HgP4Y>FwTi5{yLz(v`Y|R zfXX`Z$6Uw5XXRpfs`U>YwB(c7!V1Jq2Ai3Wv7>R5JLU9^3I;Mo!M~X{`W+5O!qc-A z*wB5kwGNqwvxGRTf+w~dw0ZU=88KZSeeUeW2(7~HC8h9X-2a0Z6^tymQX;67mOti< zUPawFIH%{LL={rS1LO+eh7vq5U6&5ye-fF!rJ8huv7o`O6Xq?8MrTEZkZ;D zMr2*M&GxqU2O!)01zMvPG3dX zn<%zq=SyrQt{jTc*%+*;CO_TyGV)XM&H{#v4dHek_DJuGtix4UmQG;E$3e2TkcG?k zQN%{NZQAA^+)*qy_E*$~0c$%hC1i60r$OY;yig*CTeS1lg`M+yRk^P04jA`&rZ`h3 zqzYid^@>^4!IMn*pCVQ9-(N|qSL|y6R@GezShQI+VNU<8r;<-n0_HHhsj@<)wW^xu z4OCYNR=vzSLRsm|_YD*4JHk=Xe;^y<#&Si0#*~#4jd$qc+=4Y`qiY>jWFE1^iuD-} z?AlU9YdNu!4okS%=ck*rCfODCI=8wVK_8K;E4PeTfb42xh-BW*0c=kN;bb~;#Bi}V z;3I3boB9A68e)J4Pp!bIk!egU2O$59{87o8()k}k#ic?>PT@r{I_=+i-1I91Fs z34Qj>buK2hx>xDrp+xHmd%T!|vX2iJ8h#MWf3c3xXix*HtzsXtbD?<{ZSBiuBC3l| zMSGnC0@>-WMJcxFc?i|)jgx~8I@PTJIX0HQ}R}C#&S61}g?7nu5dj%eRoh?l5%@PpfAC z;{TvrbdasM`1-j<^_$7Te%xpq5;6B$#q`0e5rfkL83qp7S9SCXdyzw$%U$%4bek2G zymQ8;Z`3{CVZLe%zIr56|ZrOtjg3H#VppjqKr!e$%i(lrd5&bNl{@ znp1zYKG-0&vnRTv)69a;uY1&T&`q`fAUXd+-B@7HI#}V6);t7?meTgln|E?ED?^F4 zfRo3TT#+Uj{f-mE>O5vnAeN6+(ygY6-|Q}Sc+Q0e_FJH~nuZn3>-a@nb@wRC^3FWF zRHA&}u8NQ|GnI5X-fYZgO#wd8qT$?OSwLR3=ITkqjC+w&_)YctFUuN$q_t{gpQPQFNnJ9w&*Rh6sur%@7j}Q_Gmq~Vd5xT+-1A2 zR7%jPjIEH1)=oqb=bwjAMAC07#3EgEOxbo=T1(h_#Ff3JS7Z!`Wn0Nvzj~gaa=$0A zcqY)2BFc;+Cl?P6HQMVorLq*u%IMJo+cr&PyObGMhyiB{7NoP7V~TflJ>dn&jiCFi z_nHdN{EQLufp*b|_#Drx)a1^G3H)FSL5A#nGJ=fg)wg3#r$o0x3|ou(e>Ia@zrhYb zh3Mkic+Qm;e9u`1OHav~E-hE>+4Y%Dx#|3u64)UL^-QTq+P-sWPsk*cX`|1lY>(QK`K14k1fNuR!?9Uu<9y|5)>Enn?4G6y=!D zftj<6LF>RY`AcQO!p=dz$K!Abh-W07M|2GWJ*u4^(l*co+WjP2?86A2nw9a>b#0v( zL+smFI_vC{yMn_;3phI{Q-Tbt?IlO%L<7NKku--_QY|e?cRkqihQ!7gb1Uk!4vw!&=^4omV>o?^&MIAft zW!q53BjMJ5MJEj4#-3`a(`ZD*Ha5QA5S8k<*BVtOcoTP+O>z>46`q|XjNkHm$(iZ5 zaS(YH_N5ED)3&a;4>W7=j;=?%37U+&eTeQu5m89Nsv2WhSQx1?E7}?BB6~mc`rVn{ z79gBrN$Nu*TK5RmS&y}+*ClC#AhH+x;FR>LpMp`T7H~qS_sQ!TE8|fEf-$&8df(2n z0S6v%RYQqmbyAMQ=1YCwEm|Ip=~_)`O21D3Zv99SCuFgBZK`+Ky`{XioiG zIYBtrusN#F(3{d;(#E0dr7=4Z39w#zQeh4HFX@$@c@m6q|@My zk@+}gsF7-CK6>=r@d^q93Ip}iJ2}zwx@GD8FMsip%tjvcaZf$Gi`J-!i{Jaw(a5TE za!$><=H{%edn^^dSIn1LsN}HsJv+-gDqrt<}u*7Z<#i)E&~G zul-)D5#cM&OXN%y^z##<#;-VPjxX`0R+S-WJjG=1OVYD3yq_^tuq}E`E>&!nO<OT8wZBqJF`|Ak!(N0+|$eMyecOw)YP=FgYA~v(?0kM6d+L1PVWf zzM}snT}_$M$22J(a!uE|mKsro}S;`zJB)!#WP!Xu0Y zwlm(S-(UA8c)eG}OF=t?QGVABRF**9#&Rg9b!Rr7*xMT7dYh;+g7wrZ+tZt6;OCm` z6B)kgwi^CbxO4tT0EqA@zSSYk=i^5bP0h|K;8aG(Ce|_gYFFnzJXaNE>MW0^GO}l~ zH5Fe14M8;ziuGa=G+ssdR7JTD;uh=7NXz z#h9m6Vx3{$?QuQq4%*d=d@MdSJxSE=+FAw5KsBaT>SgLD0S4YY$edHy6Ql}5Ue>z} z;LjQyBA#27Nvf7y{@^UtmHy?{Dq(E#pwf@+{F-AB^l|(R^=)Nm>}XIJxlY<2bk<#0 zk8c!cZP483aX!Lh>mGT?c;(}fGizfwj-bUP--HPKM%b)aM!bOGODuYgNX2*;~4qoHovm?WAhuayDK73ZTicgHL7_1veZmw zxszkOm!*b2Wfd;8jNzv>V1DX&ncu`K%Z9~nW6dVa^%0ysluFm263bklNNR1)re@j7+>S)-%Zxa8>lgRCE3BTpwcl6x`j?pv_Mj;02Nn_QJFV`+{WNBw zEnV4VDW@&tN_v}BL%#9h>f7|V#tii<=CrpK=R2m{_C@DrXU)n4+7XRPbf0$?3_JBE;)|sn&AXeQQ~$qRY#BlGvU-eVX#> zGIS%S+h&(15&0hKqj9-z$=c*Kl~-jvll#eP8}8S8nFDxFkVSvHWv?#)drx4N>m@nS z>rW!(N=nEsyiy~fT5K{?)Zgn0yyP8xV=n0{MhB-ge7<## zIVSZ}k7&f2>nWL%RHGUe?0VW~Xfij$2h@cIcDwq=ADjh?q&4By}YAdVP zEMQQ-9E;`H%%{7v#!W9KnTg`3!Z45A#Egm94u@=S2Ii*Kih%_=^Mvddg=T+TPhu_t;o9O7If$VY_c~Ofa z51crZ_955}-Te9IUoAOBJRPlWKOOQm&VazT&M(B2<%qeYi|V$52pm9!3Yo z+NB{mY{w$LwN<_=oLcKof)Ro9y4QUHk%oK*yLjR3&6_tHuC37SN^6|rut^O65z^4~ zz@+|Ho$9875ma21PUDk6u=6L_{WPad1D^B>f8LsOxA|BFp}1vt7@I&tojTAt`e&zk zr}Wf-TqZwKkz;iLXno2-lW-BsifVO$q1=C5)*O@Is5nBSurel{H{hVCJf`A3C)#l@ zf4*^t_fkyc{1{ZJuFKvkZ3)d6hXULZc|`serzJBzpQ8NrM}GFE8+CXKkl%Tq>8Ts% zKsFhE;3VNX6f+=giO~BBY2p%$8C48b6#UbKlEuj#t8Hu6mtZ0h2MX$5=a{{qAE7iU z|Iw$Z*PKzAIcFr-9-UcRyC|ya!-o#bx;~sWa7U81H)|!mH_W8YU$IS{;0p`W|J_!h z?NghIbIDR5r%aa9py*$KcQK9azlxbTXpUq_A>@A5UrQBfVB7l=dWP&UqA35m11B_` zOdTu2Y34L#9I zuLiRc;>Y;rW@mG@N&;iAo{~vRE^(V8VGz59B0Yfoqwq)^9^$d^r@sPFK@;+Dq_)hc z93(Eh^-?o}zObX3#BD_bU*mN$+AV)p<=x9K18Qz=1<RTX`t&vE#mdyr-_F!+jddOagF&gM7lojU>93@LWQ}C_yLac&$dpJQ#>(odfzcnK zMxny3An6w2{Ja0^`dmZ>r^K~Pq}j-KrOt!|&9~d>LS$TjF4Dr@g!k`H*W;!8@&ur8 z7^>9Op}3qd636zVv5^Qp9a?H?wuWt6O?klHGKJI)=yHd+OwGo z-WKF<>%MP)zPnvjT#2c%_=sc>1dZ{MZ9n0!iVtC1 z?hrzXb?Xo`rS`b`PkplPdxr(o)Y>u!>rbkZhNoW6Qszyah{uZK-4qK-5Zi4Gw1z>t z|NeGALzY3s^2qrfse6wI=CP6X?3b@0fB%X1(g(^QQ6k44Fb*Y_&1EcCtUlmG)(s#U zizGI@mR_`_G)e1#==>>6P~J{5WqHS$+6jS>AbuM0?cg1#J8qS)@Rsh$WRrBrp)T6| zbeZ?%Lwku$hvYj;va_=z85wE;A})qB+X#fB;38eYvX2jP7S>}m?AGH?)phzbg5*_u zpiNvzP0jJcpP={9pT2`n-@BvZK6RUkfgiY~7f@>C?K7-=F~#nC5?w2AX~c!>ot|+k zoQN{ygzVq*7tbMdc~gllQDl`C_#E^y%qWs658Ht z{{UpJVUcear`BCVw$qC;ncLq-G4{$HM~(rykG6AqsQf2e?0}Lml)y$ujg@T4lv>|* zw@#&>XH=K&3giBS?6oDa z(Z^dLZVYzd-1hcmG}LKOc^RlLRLIFf|L^T^$g$7zH1o+Uj%1iIsa4{P`G6G*4O=46#zc)RF3cOQcF zrWnsk%mfs}xUOE9Z)`kXXQC_HY3bE(+0rWF4qd6@Nr@64xXMQGnkaZt zBR=S71zFxL9!QRwLY;VB5~qC>$|vTK@`+zON@c#(1jrZNne2n?T~hA0UnTqil}h(W zPWKEiZygB?gYLNNz`hnTzq^#YqgDV^4=5W=`~1qA8C*JmP#V>*aN)5Lm$QqDz%e$v zj@+9a9UUxUt|`#eF`);2FulI3cIalfpNZ$4fsRfRw90cs9~-nfAzdzTZ2=)(jxn1u z*;A@3?mn#r4MJ;cYlopNP6|uaD$0V6Hn?^rHtu==srtA>t0f+r^z)~Za-e;C%_f0C;$2LaN|C>l?#vC4a2q$FUeFc5V2H6EP0XFqPTY8@9mp%0CQr&x_(p=WT zdA6;3v=4MIG$dWlFJ0bXWe6A?8q(I%x@+?~JUq#7bF~j?i?6Dx3N+iK5-X6$hcJ42 zZ{RG$aJctbMx~^j8aBb{<0!THNFiH$B(1o4`Jk-7>vCl>sW~cNe{0K?1c2`HaMUs0 z)1pjFOhzr%WDN}sq}PX|{T8%^LoXOIc8_x%pX|#y{N`HCUUO))G7G!{jf@0n>dc2v zF>sVs@(IxY1BiqdIzPeJ)rRhLxY}T1*~q;o5qk8X5Pf6W40`>~adCay`-sz;m}L}Z z><9gp3TX62+0)8VmzAM8Rj^GOgj)omsl_9?n1K-Q1RbCra^XWDm4}yCu!^IR93C1v z9;R!ouA>t!wf6fu{qf5gZPEZuTZPMMGBc2#7I?J^X51ehSe_k{^?-UIH)Wy((piPv z%y!~LMt}cpWJ2KRFwUvoG6S%dgAD3y*adY>dALR@Gl0!Y9*3NlS6Q{HA5{~kP;x6R ztCDARzcM;B!~up&^T2C7ULaNCD>SSrClWU&RQ-6+5YFh~nppx=Q|&@IAv!!ZmJBx} z{OId50sI3kSV=5v>+1yQE{ul}*pVfKNKvg4Th_PN45$HWafb)s{)o#-x^?ySaNQP3 zxKw8hl-?w{HT8AE{()!SJ?sbqG$9aw+h?jL&NoIuQ{W|Ftt|}zjK6Wb(nw^ zfLrYXiExWmx-<4wZ4OM82cgle7IPJxyslvX<9SMr5JciDt@5m;k&S|yt=g%v_HCbT=FP=WV zPymgVOl)jftm1A~dmYZo z)0-MZ_bVr~jr%@g;U*EzFx5u)x#1slQ*iN0f_Yu|FCp6oQ9)Jo54HFOTk>~jMl;Pf%>cjWooI*K4KTQ;m#r$8yy?0QP?bq*%`dA)R z{8_M2Rj>dGiXe&<6$L|=DkY#IH53s-Z#H;TYNRMguOSraohUqlf`C9m=!o=$A~lcz zIqL@BclO?A&z$qeo^zaWMj`dS?rUAEeLt%m_VglX3$1GX_s-Gjx%mU(;fD|~+pm9L zF<{FA*W^BrQ^0wgg!|khgj+(AgQW*On?y4(I_4!SSX5&oSSUrNpC6kUsiV6HgY3vf z02e|C1~;!ZtQA7U@mXU`AV?0luP!_EXTkjmhE5kO)?2|oulXi;Z$!ggy)w#;{2Y{x zIzSG(IA;alw`gg@r$oM3!U>2YcYr*yV!B*~`?Bp1u=b>CZ%^2CNDvMCKVuy%IC3zE z5m)mC5o4htzgB3`5{I^qpVqWlnH#CAMxLCk7=&xGxPHW7oJ zL7oCua&nP@yds?7v)vX2)^ME_$012Q))ED6z##M&hV2d~6mn(Z?5cBr@Zf<5?9M<> z`uzMQhNHZzMs2`8DfE8}9ub{P#7-VayuWYgbsvl6x^N$&YCv`U0HtX-w0ZUR8Y35@ zj$||K)32m>wA(2AQzFRG;(PXV0ID z6i#&&q&o(^Da$o|56lv4pq_E2`S0Gn`x4p|S5sSCWio)y9mDpvkl`D(rJ%R~bEY0E zomx|KbNh}RnXr^5w4$CV5|8;X|7VHX5QGOoUXpQ9Y z9|->VGG%u;#yTFZ3uV4?c6K%_B&6jO23s;e`io}zy`?3$4f`_FbB!uP`1$jv4Sgtw zfGD}`p)bLYret7^SO0M31j?p0KNW${aWU&DZ#eJGOxkfo>ZM&jY@YM`qQz#51^qk*k3!f{I$r?L@ zOBs+^%Tf${omUvgICtT~ohs&Sf|=WkP5oyn2gI>^jlrA&KPfyq+Gessr}HB)5&>>> zM?8R6DVV1fWXOROQ2T z2ihy$I^aYKK$AeG7!Xby!c?Q?0O=a}kNW7#;E=4&^)+ZMB|QUTkqvk( zTykEeaDr!x)+6K=5TSp7yRT`Nji@=`LKXcAeOW$c5@8kcC19|Q*2`>5neMAl5#ARM zJi=V~(H)#(Fb^J}OOp1+>q-cmRMi{q4}1;D4!r8_QpaQjp#ZkxS3>FA@f`lnH1ZEBF`)G6QltWTLOHfP@nrcnLxiHRIXfPfWm3&UMKmaDS3JxElz3 z!lL=Z#Nsj;j&j6_7M#Cxfxc!7zCJAQUEt{=IUJeY^r}*&l(KjhXv~+N{LOYQe|1qm zXzGiKiS{EM4LFue9J216Mm6*n7_IH>_t2%>V*xv5phUkEANACfE?p7VadWL^YD2io8b^ef&2P`NltOht^*Cbq?eR%n@PL9r$ zWoh_UPtuwDT4aF|{6Ij-dekh0kXlc4$P_u#lNc|*?B^fElUmjyqfcl-T1h??-S+#- zSYY}G_@K9;4i+ic2gGW*pcIPqCx zYaOq}U)#3}0rLp9AsA!vMH)vci0Mz8nhvlhCxfurQnhasc(1@BVowCb2L-T-RXY?M zh-<|VQr;)ol=K8)^Z;@ccM)A%HC2|QXXsFVWndg)qoBF_t=vWWf1)9m&OCjqH3-|B+b*2Uqe}0>*2$bkTbaVX!=r% zHT<0jZ@2dP%-@}+7Gz?^s;=tJziMD~`HV;=<-Et-E0$!<8!5eWm!1(2&!&BBZE|cM zx3AQPC(=Q8?`r#8R?~9-_&Z=Mo;)~d+g=2_Rj>Y^T~)=lx~V7Hs=sS2&iy(PwB9?| zI-n;i_3RL2STx$XMIzCyIzCZPK-%!-_HA6s60dZJ)j(AKbXs0iv-xfo=`0{&~7U{(!`_8Ys9he|~j3iL1B6gRtNL%bQ z3$gC6bO-M_Ibs0C$_cR8Nexw11U+*qBKOVku~EQ4)|QS!@Fzl&0f3oZU%4(^ zalm@QNoQn0j+V?BU&jcc`p)F@n4 z1c%kmr9*iQpu?JEz$qfZ&*jSbZvhyO5lz|2mJxkT1~zKi z{!{4ke&b|{J1i7=(zJ!V{P)2iZ^Uj2T7UhxzpnbbY(>!2KCR2#J+48_SArArEM*d` z6!dRUr8MLTWR@Fcml%hoC9Pl^v&uvsc7wqs-xDY<`3Fh+_W<@pK!$pBev z?(6J`?~^C_G=%rC3cb?6DOo%J6+OhR5U@B`O1mn;ND@q6NzldKXqAksqNh{@9nzs3W&rMhT! zprZk>Xhzbv1@41DJ0O|>m&OXX6&~Y>0i;4}Z2{Fi`}Rq77uoeJMXToRWGfXnPfhh0 zEz#i0O<%R`=2W|;7jeX7^-YbyBvt;mp10%hKjLleh2~EAqjK{f?58{WtV&j@)avfM z3*ugf?wL;OWB*CWZ__eMY3p>9Z~f=)MO;JeZhA&kwo$)67~}S$0**-vMl34i*+gGS zywAn?i$m`$?~m|R9cslFEIpdU6@1iW(#^TFr_kBZ*@TEGaLD;aXdUwu0t43#br`_K z6xvQ;nKri+7fHyyMHgoNxUs#&#;#Inn|+m9Sov?hGI|t0_al#~bLU8rP2V`1k&xu^ z8l*QxwG}vmWB{?&a2m_(>@_QT<3r}9c_aGIk=5yEaNsB3UjpP>}n8JYEr zJb`jV%dTq3ov4m=O)aVP)5!?x4+zVU4T*@3&IEQhkVY{7{y2q=D&chi}DL>`ez;i>+$zWxF~7 z+#@bDX(egDR;^-i8~oM~{1&0(xJQ2-!yTF*x|k+3-KOnoeC4w3GUaMjIbs`qu*CjBTYAuy&JucE<>WiA~ z$e9s(tRZwCOMXP4*@HfnerA2Z!RA~qk`BBt3my;`faez=%rVAV6$0vc1Mt}sCfMgX z;0=)n@z;C>oZPn8csR_el6}@1d{v}?6$m95@E~F?zJ{y9ZR%z_P>lvk4M=t8&i)kl z@stKmR3`^OWjn3{1F5cfjBy80ewP$4pb@eS@UZ@1$TB6Bzd99?zZ8MP&fRllz~TP= zpAl92qpYbUJi_qETYBqx$N-?=U`R!H?VO?JWL92NJ`K3Tx~06@`XYO>r#^WAU$c9r zXV5mF|K2)-lTL;v+GVW+pHiz92mz#($Z-)lV#e>9uZ_?|++8TW{+d}{N+hh!CzOL& zV;mBuHs@u4mTFg-UF6&|r4T68A$RSgX|Tf1AhWww4ZdN+z)Zck)c3e{Z-fc0XMRj% za`&!1;c)l+x=kDONslk>rk#tGHZPntuCLb)9tthC>w9vW?ma$#&a6oQFCzni9+5zv zpozLLNJ3`k&*Clm>GZd63x|e6AZ)!hHp(Zf*Ecb18^j%>=v8FZm0zdmhirlj?p>id zmptjyl4+xJ#@wS1!BJ-{v^R|L?}O9@1m?DvjUvYMutEO6014>|?CE2xJ$VIzg$!V`W674?B@SW}HFtPiKf)V(o{3;xo|(6sLO(NKkDX0@bKaz~^WD;UaFT&D1b}(#A|-bB9cV3gD+Epl z_)1s(OC%zpN^Wl$X=b2Np5deknD(u z>)2e=?VfuaX+Lx8NM|ff9z$t?f=F`K?lkcyb$fjVKgV>=`}nelmWi)p;m9zG@#VX< zNJV1??_yslbq0l6(sP`>mj^!8IXw!zb_XX|pZGitPnA*F4UFs9rCt7MtG3aKLF;)J zGe^G2uibEpcUQ+r%qb-Y3AJ0uEtOL6OxSN{1*WL|M__mLwWY^D^zlkh9SjqM&7cFD zR6K10IlZQuR&{(07uq=K6JgZH&nKnSRs(xLthJapQlA3-RtHUU~!XN^bI*0MT!ykTk zY)1QD05^GNDT6?rv1)dl>Nb(Hdl>Nh@m@aZ)z?h+J$tn1e0Lc=zP?WU{hWMFt!MsL zflj+x+AOqYW0|Bbn_p(@my-T;-!DBC$gbW_3f(zYoiL=7IdMbV=ne*t;6!Zih(6to~@h-(y<_m+ z0l|?0igAWJ{yMs)8%M0nz+xz81!UEj;_}%aeRQzya`Y`F{b|!-AY#glxHy*uJJh75 z!p$26_Y&J8|CToFBS;vgQDFXpR5o4fSc}-I#%3-8To2%isWnE4QIz<-Q|kGBd#OEy zc?cIJfd(1GB}sigoZULz4OlLFw%fNelh&~J{2RcFb~ln!wW`M?|# zOyHezW8_7^H=>xmdiBbL?mE-o_wMoqA)#-(274i(1m1|ZFr7;eyd!0Gbv^hnJ2-gU zbXy88mu*V{NXP5Ro|R0d)6P3jZ+xb5%l^z{K*4I2UU6atx~930G)EG?;t|1w^sxF% z7aRZ1nHmC)f_ttJ>$cZ4K0zkLSbI~}b`XhY4Y-vlC6N%G4>X9Q7F?CV*1l%Wn{)tGN zkq2Z{1KE%~aikz5-7->AhH+y%cJAb6fpdv~g`paOaM@8>dU|?AjL+MrF9-q572%`B zx&8VZB8nzzzD?{{`AseNPeb1>Y0eq5GJrB~Sm~B`F+Uv!k06JA8XyUQq=*`M5ZX9` z>$8`VBj5tVG@tL-3n5|BAMi3PHDs1S@pqN$YQ9CaOq`5+{zZ1A`2zy<4S+6$mB~#2 zArvLFz|0rOxqN=-lADll9YH#)ekY%2pm~w=rP%hpqsH`a@GnF}MVlp}Z6F}4V-ER< z`E@_GHI3d>QH)Q!i-~}o0~S&Le$V6E!92O~GR$(XwMHfBv`V=FXM2+lK0@QpoWJRX zc34!S9~(CCp9>}-y!3NmzLFunH* zd}Z~uR-B(YtWWJlwZ$Kq8de z@HvdiJYbJVnZp8aV|^yBbl20Btt>XfiYY?WWT$5k92a65KBSKu?CnuYm8(V&x zKxodvfPMUWkZ!$bhepa)Km>+#>%!~w_ppvg8W?`X!6%c!C+l&~JBD%!N;^o_6QRmO zO$5pXu+FD>A%{+F6SZDnT}j)2JE|uK7Lv1LZg$uNQy+s|aX^Nur?8Xjv8tQ3?tw^51gltvD_oco^>w8GR~ct~JH61ecHn=f zO-gq^^bJu3ffm$0c2y``;c{)c6;xrFM47NY zB{ZBCQ`v9EdRpSq$IthG0?%mA9g{? zFXI@H#utC~nqZLr71G>L)Pu7|Y*(nwr$wIA2B#8s_!xEeMBSB~FkJL4qn#m_(6;qE zA9D@>xga9(8c9@5f9)%^!YGkY({>#l)(3<4u+qgrN#mU;US`sOxu8%t3x-ixCelpx zCB?G$In{}NXlVJ5#%xJ;r{2*KsJpDM_=65>B=Bp3;F4}RFu-tnGC?9`*Yw3&feUIVF{)5 zj!2(_X?S$Y7RAzG`y;KsrPd^^gZ!p@LHvf&8qE99m~)UV^WlgUfg#g|G%#EzFXuj$ zhBE5+ch&aeKN8c-fIyB^o5G9Mo=&F#Z}?F`S5~zy8=D{s zq+wQ6469={O9RoWG{}bDi zX70Q+gAcBMbY<9{Qf>KVJB_=`7G(cD8B(CsDE@p6UA&HQm@%lv1LT>EsX$6r zZAGD?96?Y`MEXWxhQ{JVd%*8qpp%oBoz(?(1R{0>6hNA`yXjeQ5*EKqPemCEd~2s% zjYr{bIxNZQ5_}0Uf{4;em}ma*fwG=dIA3UtKOKvU+qY=(LQP)U%yVU~fuh|B8JpBC zS`fLCp%?rUC+1Os5PU|E9uayz=6TS&;t|Gm*x&*@fNm8XC^c{3_A?9n5G?j)6%B^77@(nAr9a~JicmvxvCx!35wtzJOhG+sGhLZZ)ef{DF8xJnH;S4_PauLO0kMt{{FZz?(KY5BUkvnm#P;ryi2 zpsD6>!hoitA(iav?{Nm2@=Q$Upkv{U-sGCLFj|T0dLP~Ho4cJC=FO5N4+ys<*i6Sb z&z|r}E~ihYT%>cfZ}3I6vTZtta;YByonqJd@dnPBf_ByIf{`mAwu0I))UEHKrQyn^ zs6CQL;b3EhFOO9fgD?91p66Lm8G>Y35EbOxNwU%gU3;hf0Q0h$=fEfyQDc!rXA1go zDboUVx&%enir#I<(rDonZ)IQRK)lt^Gt+On&JEQ1zu7ob(x-p0)*nVgvZEYIY?81M zRU1vb-~Zkf6&@CuX`Wl1B_85?WNWF9m;O?0$GJ7B!TmOU>wyY}1q%a=oJkPo;&lVB zXC?#0`VL0T(*FhJ z0By`}*pwH$_c12Unf1V;{6ScFmp*w~zNcJ4%H?%9uqQwddJ$2qZs_YeI(&sdP{hj+ z>>3w=>jA2<480&kEDHZ&4;k5bpCr~}EGS@ap9vNL%_GQ#g;VDHB7)<fm}QCZ~#JmESF)6vZg38otJ;U6jF{U zJgfWjQ@hQiX&V0povo7N%$bz7lQHb_)`|1!jb`DsJ428MlV+CuFYyY?>tRXY;0dA> z{P*BHZidLO&xWXtf$Uv&ZZ}73=RkD|M6jn&04oRwxffy;c6pstTE9gXbtBvycY@Fs zuGLQXxv42{H+^7f2+<&;aPW1l+J6lk^}Y?7&v+S6cj7D7TT4^@;ji&W7eP47ja)Sy zw{uEi0=`Ldt{Qoai(A=#B!I|uJho&&+@!>cavSN*v$x_@bo&cwxdTRqc5b{J9)LDS znFH?3^A?UTZ%TD*%3?HZb|l=18nB!${EmfpY> zjd#J4i0Lm%MUWz;@1;xtquyg~(~nEac)r>>t&Mi31UzTbRn(FEP! z*bfNJ$CK5s3XF}%FS%B{EU|xE-S;v?zpI=6Q4NjY+L+BYRyMXD$Cq~$mNP(3vR8d* z6na$e|6yTlyyE|_voto_dEqh^K%Q*eK_t>$HWDliV4d2lL9Id9*kN!^v9V!yA-HB( zzPJewCwimJ03d70r;wh}471HInh<9_J(dny7P1XDSBK&Thzw~Lzs~yyr?vP4;4qj% zL}udW|A&E1O{A);Afg=jQ@P3H^Yh4%Ztq0&C-At2%~tvAvRM4Y#JlQ^9%l3ZdcShg zg0rgWMw7MpW<#~6hOT4~FQ{_hxf$%6{m&#bS%(R4119pb>wkK&Jz0C~{6UxWxU09l6@sP^yv)0T0fct;@+uQS+RS#( z%ie0bI%kju!tZ0b3iiLQ4lxb_n`fDS<69`3Z4A}L2Ke6p_{`oW`PmRi^@G(2WWd!x zr|>&+>gP8O>cb6r)o;B-UT~Z#x!cN`3ef|Tj0&qAZS2!kq zj>Q|Gm4n!A$jX?0L+?di$qP*}Oo9_7=k++L3&<*sApSjvpaO(YAc{uV8}FC4d}2Aw z^baLv&dUcAv6>&k8xsWG6rNTde|?h6=6zbqh+cG5Q`2k1(Rl9JZc}=XDg+m zrcEhU&F)Y;a;QLj?qE5>n3z+cCs{|^yX`|*cn_j z#XX|CSPk!<&y>9*jZM;|4R{*F?&Q$^q>Ik-A+WKHeS@#r6LmO*pT65ZA&$<7^vt4_ z`h;8#RjO@!`eXgY%~Wqk<1-<#=jXkMKX<%7I)%2lGg9sn6=ssg_N|Dxj9qKkn`BJi z|Km&hp~JC+vQ?+Ia^JP}x`kG@OKHCON0H15@KXlZnvQl!qgq~@a4)c_>2=4BlnD4Rl45uUgW8ISx#I+(Fr1$r<@pzv z>0YEE&Fq?x;~P2Fq;{X%{CT}7e;enz3DVdle#&6@EtFRLH9oyJ9Q$qV6K~cRgp5B{ zPSWb|Nvf%|&5m7rX79Cdid@nqI;Sf#J+{p=f-z;d{q{oSu24Uf(L19pI6<-a(>9Oj zm0t;4zT~&53(4X$+D9!(_1QDd-m(fVx4Gw&BR`VNMP3%qBqX>@kxnI5$hgO2idy8EI$Z_^ zbuQ3X4ukNnyZxmAEh*3$iTMjfU@_n4l;sdBm$~om zQ?sKtGqnY96GE>TUZX=?(kl_-xlz6n+Bj0@py^=fL|{99s7q;>d@kYpSjhB%(IE3G z++=%Yrc#%VjE;;id^-t2rqKLKCUWoHUMY*Ho<5ddV5m^qos;qVg`Ur$^wTCmWT9We z=wqE(ZBOsq9sJv_>tlBBm{AEP@$zR$*ELcu#a8Q=R8Nr^&3?pU%=%lj@A%-gxG+IA zuQ`)N-p+RFfxEjUtnJ(L_+uG9KFnOQFO{W4xGZA`DZ5h^HYtEExz<1Bn>AiLjW%S=o zk#ZuyDi79Hie9PSH~qzKTM9U=Tp{})Rd~s=Im5@h`s3pjq*V}UPA!HJ&Is}5T>GbX zZQhf~Kp3T4IP}sa+vihj(T4TtqRfET&Y~_t{D^IJUKGP*PeA`pyKC6ISvQe z3+`Fvro8rxcKY2lzCKyKlfVAnGZ@ATd!k6E0+ah=@ZXc>sj+Qw_ndzGYWs+)(i)n0 zvM*Lo&v2GYc`QIF&54_$Kk&*&8LgjSkvNj8q!VXf!n7Ltz+-0`%_3kVBXkDzG{Rh$ zt=7JuT6=uh=C`9Wgfjd%E84;Cnx|c?Xgglpgh>UP%91 z%`KV-EiRiRB`!Or(2{l<e|J#`t#EgvezEjdtodi+P4JkBCrJ2)Rj#43>*&105)<>nm)Wb!?)1(w6~8;r);tM={@{RMm93_v$ir+80%^UZ)Q zOfE4I+S$gzJ=hvfVP&OT$ecqN=8y6tf1UXUaZiFCUklaDSpD9f8b`-aK9Z^{1Ah%| zEq9CL<|~u1H8zd)zZ?cfr81Tu^(9ty3;MM}TrAOfuWoqX18q2Ca4Ex_E7IO`SseDl zZLq^_I#c4L_gEXN&0G-WESb7Kx3r`OjapD2ImH>WJ2VB8TNSA;XnYu^KD9=f`dE8q zqqJ02_n4`G;Q0#p022gBTnffv<@}H)+fj~b*5|^$q}#g*D$lr~9vB0qH5hVjzHWuR zD%oAnGC}ezhe)EGQwlxtZ7#|CMGC@_ z9-=I(G~7@m3g@w>LP^%Xs>f|Qv;U&uV)wxj6FQuC$c|j?!JaE=-yJbOcgaBo*Zo%s z>5T&`-u7~go|tL+>HZ`18}?)NQ<4)COjTyJNx`>q&F=gMF+$FcK}4exD);ThiA3kRGAZAOTD_p0)0$TH(0<~KgQ?iiJ8~n% znjS1sG{ckixt2+QTcqFHJkPMuI>$%qR-=4^+t#)i-`I2o=B2>-NYQ&j4)Ld>7Z$ud z=5qlmE48PhSto$dys%B4O82c+2%3yb4PKMGs>?E(eJMCG{VF-IyP@0k$-oTDE1=l6 zGwol8;+W>f#&7zQ>9it@XQpDmtWIKVcljD!17%pXpz`vzf2O&n6S-uzEJuguw=+-o ztqMK9_Tp4`PD6+B^w=+nS-t`OobIpQw7I{F5-*9@99=qSI3t(X~sqWzOg4u`$-T97mLZLT%5%e%2h8% zU1RJ$I^OX>i(0q*Mw%6~Z~X*{>N6j!=q8AQGnFdt77lAJMAM9(3$ci#KBhjYJ=%{^|r%EqTUAjXn@4xptr_SW*PC!TP2S?6F*K-Q=N;iX!tbUZ}S9e{&o6q&5^V zX=L2%Yawvv<4C7Z8p6RJfjz5stOBPTls}K*^tx*{e_v0>?Ivt(6`Xz9+73`Eb8^Q0 z_&K;H)<{FQ4jUgYbsd9gWQG@L3fYF$&2FMs9iv8D7@7hze3|=Q_ew5B?MVCOV{ZaQli@s|&dkWAUjBSu zjq-$i2K6?sy2VX2-}T5)`Pe{<|8i5g<*#wAdcL)bjh(K{Qlp@uMz|T`OB@akF-k%z zbb(2^&|53UO`pijOAFlHTj&#WIa|!+8o`p7^8!V*>MIE2uoESL9g!}@-?`CxgoBD6 zqVd=X%SYzq%W!od7N@&f!qiCqD;XOinXR+jZvG-Fj}Iq^<>?;QuSokD*DHM;ss|p> z{<~1KTkPlwoX%3f+NV?~DP{jSwgGTy0X5YXm4954g7d6nXNwerlxhRApfpZtj7 zZoJxKmA)s;z0}rpOlrsV^3a*L+KX*(Ss%ADynBUPDoOrq6U2tvsz;v2)yeXLR{Wi#Lj+hUQVSy zB&62JH+s&KlShmMD=*p=-;+5)JH*#@H~ZPv;gR-ROznNg2E%A^re3RRh|jkac-zBs zVnDPDV`~2bHAX5SYg5AdyMOByyJX9xeG-j>IeE^E;jY~#wBgxHBf&8kr=lt?-dm#Q zPjXogL~WN|O4hcf?LH)(@C5y4FE@qq&8*qRL&~wN6zeC5f{|MYoB~c3CQ|i9S)wq=C^RI8W`JjZwi)mlpnQkN=@= zcM5f_Wa{Elh2r@3mxXP?*%c8nvFUx++J^am=v;f-%sa;QHyxa+B69`D&QPp`()t#) zq6tEFony{^4`f}ym?KMyJLrnf_kT9%d&2mksJxr z4~LIa`N6AwygK^X1=Jk-Z_OC13J$?T!V|SJih#8 zRnzal6zj5w3JL+=H9P)&Dye9@O;`5amlgk1n_`W@NzX(%#RRX7BObZ4d4Rw33G#J& zV6J_My#v!-$uR628S`7^X^?R!$^^>pAY|SZ3|w>JtJOKfa^;Xbd+acM;PlFl;xL`c z!&>HwWhk6dtzz9!`0>^Y&V?67K zBaWtpQMf~5)ze#Nbxf?H7;8Qy{AP3v^~pPDT~#yl(?^6vro!QLy1CBVFwMSa{1^WF z)ES+3L+3bmxqDFbiArm#Ob&{pfZ&*L~^Yto>N< zcFlQ(*Uh7-96H9ay6#Hi4PJ8I6g^b={aredI9{!Yy}g;*ztVz^_=c&Rqm4QBoex+t zw^~_cp-GD;b0=(N_{1xTdHp}s1akFZ8NfhHK-fOw(qnwb?U7OtYn~UE0)7EzJ!{`7 z)cXKm@t2kEr|QPRr0GsCJ}G!ZoLRooFq7g4GFvXie!0;w+>%K1%c8NJmXd1jL-Srd z{nXxUZ~WT!dUi1e=1Xk&k<(;7zP&UduXj-gj*JMo->%MC)2yAeN8 zalo!C1;bd3GA*!(T3hVCb&K0<6as`BQgRHfvgF6|@hheI9H|rFJ|zA$0lJ-wLje1H zKGvCKvi*Zp(zF|3934R_xn7gC7L|}*xgVQzpOBA{FOx3TBBBXT_sMBfS86JG+}%7P zug2w;mzn)D3>AIVvMx_iD%V{~EKOPHYEt&2Prte{9;D_>{F-Y2cURnweKWa5tTjXM z7%=2HEZ1Xg)1trag-}%aloGHrC~2waoO2gTj0wmof4%zw%vBY5!SHA*D}vi}5)YOR zd0O{>h+Rj=PNR0xQ@KNOt)8q@7c09+^LEESsN7lMnu7i)G*8iq+LGe_~Y`qr1fX3GQ3S!QY@Z= zmVJK|s<|~D(X{u*K^N%cn8@OauUv#6q`GL)K5OL31`)Bj24arz(+EO!PM-id$$XYo zE9Ib?5lN>;SFC4fn0nm<2cgKe!3~MbSBgZZ{UD`DO)oq&b+6zi{0d?~B+C~VilTs#K@|sI_ zbb!Bz?WgmZ4h0jWS2yhp4Ohp^=q?$$ZY}KSmQ-43YhAYz#QOL+f%;<)?mjRHJ}jrj zqZjl(Z#Zds<>Bq6$<$_ax#1GSyeq04=s|hvI8wfy)8eG7ELNN_1lJ>6F#BGmvcNP~Cb&02*q5i}+;atm%6<2lrSwoR zm*HP2`N6|S>BtyMlWNqq8z;tme3tJwpD9nuLvOF&UGzJiKT5iIxJnHDQR{77xqeT{ zp%9L&J^J;XPYY`q5O_c^A^ZTHbb#0;iYi}peJ(kf^RCgQ*E6R?c1Tn%;0!#c3_wE6=!&3M>@+gc^-r+$qHwGKfJ!-hGz>yj=pCGGxK{ zFw|`m{DUwox0>#ac@HETe9u9#Wu7fsQxS14(>qmNnU8m|^*#j?-Q!T5)>rXbsj7cD z#^{jK&<`6U(~P7oS_X+>oLh6KNhf?iaF3i%M;y8|>)t+I4whqK zO+TInv1CW}B{d^f#jW5-J0w|(`f{RaaVWk#gQ4ty14ojPsQjLDvfZ=i-1Wid_S%qn zu)DYFU(K{lG<2rg8!O;)DrRP%RAe_mN-RgtRVE9P4@iFERQ-8vaCO6D*PnBwA7b@n zweapU;^Hg2wfMTY{DqeB7F=$8NlkLMY3~<7*@b^)8I3dPLQ>Z>wE5h>EEs<)auJOL zk0trqI2e;J-i%Z*Kw=Pq3j7-_Pg5`>#p=l5rjUSf9$kMP{jqJ=TjToM^pJ2th+;95 z@r%Da6+aC>f#77sq-gMRL$cUDPt>UyS+FjAM`w?qzI=|w95TKA z!g-fCUp&k5C|_4zX%E5Uw1*z*_O_J#o5|hd%VGK3QoycrpJ@DMX70MLXu9jpcKK`A zYt9UM@A7F!1dtHx8E7{A3aIcGe5OH-|>!7gL49>^-G@Y8h>7= zBT-_g^7oCBHEOvsr$g;2LKi zF+lo7fN>jzF|}4gjlsTZXlia>HQ>*r`|-^!Ew&8|cqFCZ zx17S09Km_bq!jg$X8f%LdW*Yk4Ht+~L87NAj+6ls#jUE*uC1LuvE*1neSS6mYfs55 zA5T_xLyN4Q@O21x3)U-~gY?duo`Fo(L;EtlHT!=qmlp}w^iWjZtkctq?fUUZMq`_G zKoH$x{45=9nRSw?Y^8JeltN@a5_F^|s*y641ni8W)6L|iA%EZCtonBJ@=6gVBXnKg zxJ{Hl#D9@wo)T3oCYk%zjWKLsXA&xavN>oMrGLFEOu^eo_ICeZ^5Ug}aLD~npl(N< z>#Z5lo`EcHl*Q4L5MTTRIZSJh1MxhH%Htm4Kj2h*8!hHOZ$DI$deU>Lt&bi;%%DXW zn1dOv)oCnm}__>hY5 zxhHejr12CPZb^x)gqdo!4i$}c1>8d2uOoqr!N#xIHH`C{&k|UBL};sv?=!i$B34&= z`_t~IF?A~6H1d7`8=4**AC~=~YWI$7IYWZq+ADnU`>|sp-(9E+uL|aVXXy_PoxLt#j>5g^odS~W@O=i9gSq{s%FX<2Lbk+X-JU#z_mmr$a5|fmij11={ zIOM51Z)`!@%^{jHpMsyxwGWQc5sTXXdSX8M@|HZ}OCifsM*((nwpO1;fxMilQ?|Ke zDJK1H>U1J|;y)GJl|&cqiW?8`p>#dtAG_b`EjH;&^tXbaU462%h6DCT!&w9A^W1I}vIwKx^JH%)9z5asq7{C62UapoT zl1#NR#^W9UWCgkjFlZ|;^f$AkW@A1sz9n}nh&^tt6zhS!w~Qs4!{QV;l?^Q->rGS5 z=#%H%*N8HgQtSK`fugGDq9MEfi(kUC?xdMc{2PeWHL4}PQ4jPrFS9DDn*9~8t8Gka zvPn%o7N8T#q6zF}pPP4<82S(TPLbxL7L#)C$v(+BDTyr!qQ_;nYHDMN(#8&Jp3w(7 zVq@GM-#%DTu893L5N8g*v=QC-s3N@TYt>ry2ac2Z&KnlDIr#UbA-=@MBoP}Sm^xRz z&C?KYjmCN#7&AR(^?RmxGl86AR@iT66P+l#f{x0jwT-Pv^Q?B9LF(#kQEW)xK}JMU z4l$2Zos!#e2Pdt#7-5;Lm^KF6Ayeb`&&x8hvgAU8OPi=X7y-4>mQbV>@O4~^i7Y8j z;?d-`xg~)Q8?VTAR>OOvR=@c_KhOWKh8Ox7eZ9(fOWpwAvH6)I$H4ut8Z(%T$5tA# zP}o;Dv~jJFt_)$jQ@4`7C=aXbR`6??(k`J2cPO z+X9_u6P2j58(D?SyX_Y1X;b7k7PLM5I!VFTAaYww>Pqt-W1+W*o^f#CZ+G$VC?}u# z90ZbTn0W(l=V!CQPM4fpSRnl~5PY$C-*gxLRIWO?HBjL%RE85=YUyalJ1QozHdHyT z*`AB=z`}AF(uqa8%k7}PEtt4@^F7jX56r2}b;x`ffQ*RLq%2HPOncrI+t71GqaQ+H2KBQ!_vtv@Bs4Ors! zB4n`=q9KJD6m#J&|2e~if7Q65ko1@vsl+Yixi-3(NbS1+61of#uo9R>`&K4-+@{HkL<+6_hIx=%&$UAdcFm&G zj@p|_fCfSkjHDo}tc`hZYzf^PvkDSn$ugaya-K-XHhgO?F8^TYKy7E8L3u2Kr%J*ENtGg5x+)?N9jfHi~e)We{s1(k}{ ze~*$UZ}!!&H6J{DIPLF)r@!{}^iZ{ao(8cJ0ww9V3H^T8WA6@Z4*mHQTWH^B7$UoL zba1~a$aE3p&Y$(z1Q_~BPkld@Zq0nUlkdKrXgD4;EeO4IbCwKwb5P9n*hts@JNZBe zx`#f{dk?D3n>X#BU$ZGEFn1kk14Z!W(7w#=o2@~}S7cM&Z2#H#_x=A@1K9qjv3r}1 zM;p(+@W0)c)E8N#M?;SDqcG5G8(YQ+_IBBzdbOBqS)%FPo83)peq4?A#2U8`p7%@0 zBcGZgR=KQsNgXgQM(3u0r^3+et9UMGub{renL3^CUN%CjZ`@SKdrInEzrTBE7XQr4 zJar0q_odL6&MU1O>R=nS+P&rHdi3_?V<9bvx^&+4|2~!SUju3V<@vX@ z&8~SSN2!l=ivD{2Dfe20nf8*Rm>IL-u-ZrusfU`M8MLvLkQX-&bUb+42D!!XrMs*5aSr7FGK*jM9Oh7WmWb#YWn? zwquVD*d)XJ$JKN5bHJgt|JvANWHw!?6kFj8V>H>0BtlyMiNs=uCF?b1c6BR^H|71& zNn`I@t=Pgo)!RYU%H;iXlVxy9yG~?Un!L-ILJe56s=d1_n4YQpa*wbEA2r`ZW*_RV ze*UbxNIUAE=?xnt$QE_oYOL5}yeqG}$~&He^*ecKMm8(H%@+{Sq?7u^R+$MuB5k^g z%eq~y)BWT2W{G|~@aHy6==i;`prey(bsGs}4mHZo{k(!|`(9$vXknJ<&Q_$N(6WO) z#3uK5#uDIsNrsEr8<0*|rzj0|mtths_;LNS+24k&9^2{v$NL)|^nYXvX+CS8C@lZ0 zG>_kB`Sbr^;-TkIeST^7q8->g=+5lD2c)zAX?I_b)gx9n6@pgEwmeib4VXPPP*2s1 zLK^-y&lfhfvx9%^bVO8?C06LM0La`C2*4jJQdj%O81;e4v;shs4VbsB+qR83^}oAD z;$_lj^x^ygh3w|2u8a+505(oApsr}Y2WvOqjEx+;-vzm78-Fe&ZbOJMYz_PW^ba-q z{a63cANvM0VdMcH&H!8tz;MT9o}B;%m(y7DsXH4c^7AfaG$z~GL@;>>Vry5({R8{= zCxS?G6CkZ=w`saa){xe0Ujk&@@t7tHtifXi=;_dNb}sy1ZDf=sAFAVjC^`R6mvPfb z9^m4#*&Ie$^oI(Id<4J0|35UC@P7YCZYbn~v;FUF@_)aDA6M4LkDBj~9TPrxu5fem z+4b|Ahk(wHk^$Qv(Vb*CCPn90z6b8feud61BvqNq&B%BgVjk*a)F!DadhPzX9hk}S zp29)Gq8?Eec~d{b%_AZsp^5vTgg5n2wh_1f=sdq~>8TNJOPwU7D6Fg<#uQPh5Y^K;YQgE@t-tWJk;C>H{ zJX2Hqg)!Nrt;`RBHnf;qJUV+#L}lvc%|bB{^8)Rw67ynWbaLCVJNuwkAD7~|Z}kR*36|a^<^>ZgA0}<5RMa8_w+FmX7Uocl^PCe? z?ydZ`Ak85!zvewkt8>IjxGdOlT{x@Q>dmkLP6ICGHovGyNe2<6QAHA;Yc=*{+X~hh8`nH`*B66a7Z{&4c_0#Daw)k)j~Imw=+Q00BinMS1`Q>C!?I0fA7m_e+?0@9myFZ{Iol_PyPl zGe^h#^Zmbi%kSRrcNaZqMd>LPHC{SHBhB=-H|vch9O`l?N*H1B3?tXPvJ{c$;Rdut zbm8jDR@N{#-*Vv0cv~B#wf!P+)?nHi^bw zR%VrTe1871ey7&n-Yvo0K0aO(ci3A#GPJyxdUuJM(?hDdT~dYK{$vwXU>2MH>=`$% zNtKO}ZqR0?_Vu!rZD;#2($d)Ofy-8z7cR5mnx4FH|7~BnPwtWh|J#FL7eh>psb7=@ zXYSDiaVdl}IF0wCWUb5N_TwsTxkaDIYRG;5T;)%Rl0B%2o#SG>?;r+zOS@eDePT(? zc)rU7J)x~MA0c!bFqXS@-A2Sw6C;nW$|tc zr5NunHm-^AxqPi#$>&6JPDqJpIM?*_x(iv@h0_cKcLECr|W~*Q^!9IWaC6 zj*ODKn%CU=lNR2r@I;ArqvHC)C^#J!rv@rLya%`SPuV(@l3?pM%TKcl!$roVzDe9@ zU)y=_V%Ug|7E2jBks!epY%A8;I#}B{SL{Zkb$QQphSw64>UvEDaVjcyf_pa}Lc0@$ zA+9p9--Lg}EqUybtC*{>P@0`QuBr8{XmX5*!x)vu$UfTov)A{}L)v%bvWFrU%&j3= z;-s13m}Zw4F7i13gWy6#N9Eq526KKm^7%;?KA;kK=|NAPpr{YSt5=r(PgLk?^Nt4^dY z&U?``b^Y|<{xyxMB9P+rf;YIfLKS2XI#Q`IQ5m9hzy%bO`k{0cJO?aTmOSX#ku-n+~zUk3(30_w?_09v`17vS`G^f`VXcc|}E2 z(9@JG7`81KED_`F(0dW5>4SM&H#gqe%_q|;nwdq6CQdoMuxUQ{ZvWiysN-<@P)az~ zs(rdX@=ZI@HoH{O$|qgd3(lFSk1u7a)Z%rCTsuYd{e0JB5au|xpd*TtbThC+2$46L zR$ZEak-y#zlPq5ErOlRBme%v{^inL)Xm-guAjx1iPt(TfyT;lE3?(kk7{ZfHY&^pF zm@PmnFBk$}HE7~4V+K@>Id)}2Xzgg?k zx3DXvGG{>+=J_(%lF5ARWlHAtrQZ@@_oZ9KJjwD-PY2zAgKLuDvZBYQN(b&+rKW)r zL&`hi)md}|BG^%lF(_nvdHP!eY%F3@GT)LmaZig#yK8{RuXL|vcGL>!mqb+@mip|i7dD*(U9r=5+t-tB0S{8K7etlW5O+>x1P9IXYD$kmsUnf&Fha(2(kh<0z|i`T&F#ZQ0$Gd`Hp{8S+rJ+&ohTgTz2hFp}T$9$Sy60NX5 zqR7|sG6K{EnyIbbQJEnHjMLk>jy90zrHz+XAQ$XjqlK_(L{3G#wRmYlDlg;Z%J*S@ zFUGB8SQ_)ULZd~MpE*5CA}{A(?$x{{&V5tt%Z2eq5Ec}B^>u%=bs zpT^yw4yMib*n`&QHW^({%XD8Kh=+!%$YOA$G8u%MAD}2n-(0r{z7q_N{!d8vFIJyH z!$lHdyr8S$dorJO9~6_o!yOhNQb2siz*Lc5Uaq__KX0<6rpG_tU1kn!^TLM@7hcPm z8kgnT!^bWH6U=*KxL4a*C(7UdZs7jpOZ@6WNBC9ci3LKJD%#Xj(G0z)()+?V$1xNp zE#pvOWFAg0Zq2*k-tSc0(-i+m4)TBI8Av0vsbhQxaTX;8@AK|WP#jFcivKNpqEn+nBd7@mGE#Jikq}|# zt*XaN_O7x9sQ@zffuuST{O#-gpOF2o@$yzJ2r8Wl3ZPlv#UV@4seCf-I7CN>s3bVO5TBdXFXbAEElQytgQ0BL;Vei@Ec!Ouh zvYqkNyDDNLuNCE}DcBu`yhHneFaV}}djLn4jGbS=#l_jDM`l2Ge_u|P*A?cKHQH%p z(pdPhdeQfj6}>C$!p+$(ID3Cg!gE-}jil@lhoNHMrFCj^BO)UvMrUxnru?Xp_FT`z zT{2Jvh{oNeW?GCF&+F3kwamSc!>j|OEm&5TIye+UJ_Y~90u#OkYHgrmygyxKBF`b+@nsqI>O+DwF7J1w= zTs+$J4}5AVUA#Jy%4i|yIkktBbfz7UJ*2@?fsLA|XhQ`Hd=v1HQ7K~S@9gY+bY}2E z3#_#Tk8qsK=ZHw-EABy1`wa}ls`Kx_)(+z^(QbM(H^_OQF@Y`O=?d@4L|yUh$OFri zy!vb|iH1#%HT3HjII4pNw6W}Qu}Znlwtf3DyRX0;8LXPfYHKYvn1AzZb67UN2j}Ug z;{iqUF56EE3Lg}z?F&X>&^ZmuKt}a&0f8MMMs`dPLg46_SQ}fwe0J1 zx9{9}4>%pM?e5~IB9~ND(4k|N&ujh+8Ros2vaB@Ld^VWz8}aCPvQYs3bybiUufN7g0 z$8LN*tT^*7F^rrFD-^KfBQhQ3C&)$*yM62Tn7|n*FJ(DG<(_b@|0p1B7=zzA86!w``Py#@lgzIoS2qu%T3n)QJ)x6hF$w0wv{T}sIBvNsC zrn%+27f>%Xfhwr7U~i*9pT#^@%AdTZ?QGGGC7n+JYfYO7Hn%j4%T)Ga+?(P^`OT*M zVzM{#%=msaOUobk3*7lwCb%zn2N*7gU#1ZA$m(9<+$tmFE)c@$;DveTF7?}Zvcb?D zomj^oTQ{rX`kw9fC=tS0Oz`t(g%eLvCxNpSXh2PVw0*}P`@pSd)b)w{JzLLt+dxh< z-1HVdCM8v3dosOef4&_f|z?^D+eQP74^I45Xus|Kc zE)@BII|{!686W_GvP4KB=QwiZJhL#FuAM>_3l{vM17I5>1_Lg1{BVN`FdP?<1 zvUZk~ATev#{}~WNMEXfdtadO4^6JcLLd@wZzrBMAHy_pjz9>|O1kdsOQKvnW=?MQ% zz?x$Wyin(fgi2Re7wTRJbg#%|D0d&)M6_+JR+TreUR{D)@{d^;mp(H?9RwTJjpMk} z5J-y5=kw@U!>o1X$7Rz=&V!&TNsM`Mc^?bgq`fLpnErF{0JuI_1whhPzt{b#Q={-v zLxb*L%`$UL|95sA^%7-=KNIgO-@b~K>uNyelpsP3(KT&H)m5aIDmmiMh9g2~=>jtf z`X%&tET2GoobMiN=DRRSOp41NXH{pkUyb$h1qRunLozPV(=EOT_v=ACN3p~- zChkGZ@@RLKSo{evH<*hGO)GC-4cjR1Wk#u(;lY}ORcXdtsPfX)Pz(CM^1mTi2)m%{ zOp9Q7Ak?L3^BC)GEQe4GbfSh;e@i*K6`Mvv{#z%F*i)5n-)W;jk%}u~)f9{PJ!(Y) zj)j!d#Lq9Sn(y3};sLWAxbCXl95vHg1d*Wp<3Wdp9YZK-b2#fzxd)cZ{@^dU{`MtL zC-|YCRQrJE%&YJ?v#u_$6$fz<__8IrT*uSnXhVgskI7tM>_ukLv~o)gmXu#gfU%Pu zmevgn4G7sy@{X_gq{6NMX!^M>L{+wwX54CbA4_|b*$(kic*XBPc8>zQ;=#SK_@$9Us+^Nmv8@8|Fl{2(^e>JAJuS_T`6idfn@GlEq(40fR%*JNLkShI)A^DB4qGNz z98?Bf<#+Uh>^7QR&zMfn_lZbz9}*`D$eoF8E!!v7@WT$b>Y3snj>TW_57q_vtDg># z+06Uz!T8Z{JIM)77TzndXYy{0Xgmm@rSD91Ffyn&%k}iwUma~;{d{L)md``f3xhQ* zz@K^o2+CmGhW4j}_JKQ%F7vw7HK)$@6kbQ*GI0;kgDp1cBX9(OQ^l#VJz(npila|e zpX$eE#|eSIfsh4oWXX#ZC-QXHBGj~(lRMXqs%j9lS5KnNbL2LW>!ms6=`zsyOIeDc zPeTqchXc^d>|lq>*U3q!=gVHE4*vaKcxmwQmD@l4v~~cXctEkP0sQ{Z^+X3VGAQ}Bi0>U;oj_fuoRNb}wr*P+rm1c(Cya$lcGJdyTvwG5z5QSn5n zIWU&1|2*@$?+!5u=0~>m6im<6iytonETad9%`Ih2!nC4yC>6TQ(Y6T)gTYIKc~L)) zC_>_XtZG=}?5~mP#DGS;id{z-Hs7xJA|Pg?S~TC~a{X#I7~(7m9mT>`nN5NZW}(Jg?2vphm0~&+n2ipkuI%rda}cOeY|M)v{Tr?+ z{?!hd0~Nr1Bbvw{}m-Z6j+ z0H_d~=FFZJnI^-pF*X}m5$?2p*eR@gQkIVB}q`%qggIedV)rY%!pm zfYYQk;mek<+;W?2Pap3fhJvRKA&9hY{90N}g{Ri$tI2W))TYIO3lrMM%nt&cCf9bB@ zM=~~b)i`rS{q9a)gpqPFZnSOk^HV+1;#;oVdAzSxs$LX$2=htZ)^%gPSD{gmv5nwj zL7i;_{gI0{5Tz5q!7j3x?bw7td8-~{qmGbS0qI&dk=vyl-AJ)GG~ShW_PV=o4ip%H zf~^E|C}KkZ9O@iBUeyM2*g*W6U14vyGsd_SJLLg{WZ~HE!QS{erSrx%We_*(yiqZm z;tilJ?BvokCwGjlDavJp?D3dVM3#qYT2qx-<12&}K(Fh*F`EpzarOQ}gs8=SRkp}{ zK#IMl$K}FkTJ0s0uRXXsha#ndg+xNzpXOG0EI|;83S(Xd66?z9N@2i^L50!a*a<^} zI^WB zsZVl%_Xr5di`4AjZKNkBCs$jD&xa4E7Fo6D4P8(57XnkxG=0YuHBKz(@`Mg)U1xH>}_NyPM0uaioCGI@vJ4@ z#qAZg^B2g!ND5|@zNHZ{q2K5oE?R-1K-o*}oC|FB^Mu>&p!W7s%pRT_YLyq^ps!oE zL3pCZK_J~g8FEsNqlhC2eYVp#3aR)g@agBI`8*w;wK(t-u-Ra~asn+3r1&v?BkD!g zQg7Z-s?~g@B*RqReYj)0IhXA^%=0dYE&~poCot05%O6l~wU{q}tK=V8RcnwI8zKNP z)$pRA5P{jE6xeOc>{BQRM3wsrY7ZdBFsuV?KpTi=AYcQapTK2^WoNmpE4Q2FlF$WE zD%0Y;6*d!y3e}hvG+uwVxH#;FbC4nemta6bpz_Z+06q3|u(JYLLWd4%KnM;;wC6bY zMaUJ(GbGv$15EzpL<|vxIFkCcUUn%zl2d5GY>H_B8*+M0liI(vL{#BTO`;JFC;J1W z)o+0u*CWDz^)iQ}P`PO4r3Qf`3))Hge1#yVq*nmbj`!hOLsg;d*ccDMdtnl5eXL%7 zrm}J)zdZyL?@zZiv-9pH^}TSGWh*8%wG}V4_F``g$2q|*sr1O%_EXK05a^zHGsB_o zt_Q|7Am9k$(FYx}6QYsdLK0o^lgEik-?De2ATV?Wh5b&1J@K-R-^_%a+TyN{6iEg;SCY^te`N{z#j|{sU)C(wfjScMn^LtSlA+V z>?DN9ET<(H9_RrY1pqGyE1(5#f3;b^+ow(Fa2jc(P=n9+M07-8R z$Qp_9G9<#He;3rkh-SK zGsJ|!3X9_{HVpeK`r{SKl=YqN%}kX=9L}{8&jVt?y@6a8On-q^2%IT^{f%pDO9zr3 zCsmmlT^)V`NrIps&@jqU;;q&OmiXQauQRp0t*Jj1sX4@Wj@ZsHhAoh3hQ8wTl6j6m zwEdShVwBZh?btz8tq+n8e2L6Mf_5_W3uTen@Ni#64w@zk+z__d4u4jzm@HC}4JbVK zn}&0!=kUE9$(F{Z`KGLHp*eqmvR#-o!$=uB8-=cr zLSQ`sVpZh3!3Ro3JH<3oI{gF2uU)M);)bj21L9_XRvu^UNtm@_4HA{rmSrAY-j1IV`gaFo1evQ`k5W zP4{;nLK&`*&>Q)~J*Y&B^S`E;imXkr^YqEShP>V#eTTYW{5E9 zT{X-hgt<+2k)U2|*p?9H4=@-Jh))gY_%3Ibl$Mgj2K><3$ZJLKfeKWB6rX3JIH+wX zAO&m?RV4Xf6Xru^P!U4v$PtK^`XHo(339-5y8gHC-i0WarizM+ed%E1!XKv%nA7xE z({`CJ?=kz!ul^s49vJOEm8TJHxz*3#NN8g{_1{CLzcSWfwCjr8+}9Ms&jel7&bwZ| z-qKl`rh?g%d+Myd+&3fZSUhaN2r#DG8F|Q1qlfWmCBRH&qzo} z5hNsMi^$J`CHqgU^1wgmt>o0@NJu_JQ69b^1wXUCeDX}4gv67Rgyc;i*a|Fq^NWPU z?Jfz)@^cariFYI<4323H+K)g%x3!+?DxRF292^|1udnay?09;5&d<*a3kw$&6FG$;rOHzLu620)a3-KHk>Ww!6EVn3xEKLhbGCL!$%oD^thT z+9x(T8yXrafZT6CfY!0vfyJ+xL-^0h58Bswzp!~3Dt$`O1)K|xtrS$t_@T48+r`<9@zvYaoY zF)cIhk*OX&Ztn+IQ|o)YqBG5GUCSD>U4mk*JpyeJzVSWVW^|zz((;eX zdKV%JKB{Qx=DtsHfjgq2{H<-R0z!T2>+7qks+O0RKYsiOdlTlH`5`wq_xtznZ|jGw z-H`C;^pN-M=9bpTcSVs+qhG#!X=-Y+_X>)Lh zQM0$VcVuLwv9ZzI++0CH!O6*KdV2c9hY!ig$*rxezP`RXIyy;xcr$pAzKvTn1{YHE z-99q=k=$csZbN+c(%a?<%YZny@yxCs+6^DAL+YExxZH)8ygds5s^`Sps4rIB1%rz%Go=$EF{7&E-o%MH5}+c`-OPn zmJ?2w;2Co@<;Nr_QsyV%hl`F+3|&Y_uBV>6fNhzASFTvgN+$>uMRU1eg))i>EA zAu%dec`T=km{@BH&AeunvX~TegRtQJhV}fl684pX=<}wr!B?*ed_OZuHT>Y2K$*b# z*LG3Ul7p4VM=!-nmmCD@WLaGYB&L4nTbB>n<2{c~_PjI1ZnLxVL&nOW5F2`*#}OTM(JH6LTHj` zD*Fec3iX7S;+NCVLr<$#L+gHJ7}3{+Vd4w*8YG{RkbEX@!P)=5C@SK8VANZBH9X1o zV+JDJ|LkW|TzS!~zQ)IB#FZc5toXMYZ+s<*kYAK7bW&fhdYmq%$U-pk=lgclI(czL z9v~sf(ps}AV4CYeQmikVb5d;jDlk7739aK+y})851RY#ioP2{gL-Ix&mjEq~F<^2N zsY+io>{>y@jNpZL|TMZDUP>A0W&*gkvstkD48z$WnQ_Yv3$)E*Y7xqxPOe?`<4(N-g)@5=a*A_~w*PrCT`ac?rr=CyY?hb>s3PFoaJQT~Hc3xj6XdH2wyIg?INwSn(@CVJ~jSA-ye^8%Mp4B%Cv)d2(g$ZwrQ z&pba{^{4@V_W$cfdhh1G>UD|aeb4GtjH?P6;|JS1DiW06bYraGnk<96bcUDEdY;;&?|#g4(|N*oNOI34GYHtR zCQb6VoLz1t!F-iAoT8a0t)F^X+r+grMgF3?~}VmryT*r zWKKC$qqlbYc>p{1!#VR8c6P;5^aLKE`*{2x*m7bn)lu@Yk$ZtlIBHC9%OmR`mr zf#-3gPZwGx`8Y?Jom!Dv-zt-X-Z2_qHFuYb4lRy-)WmAZ)$z^TZ-}o)QlNlgK9qL) z?@lAFl1tU^_Z(Xrxfed>ul=}_E+#7T;2xi4$FQ@DlB!3BnTB9TyHg~Y=97C!un#Gy zXrvYSJwn8_1(%*?aBXh*!8qJbHC%u<>(; zxF6T`e3E{3$ni{{vokkW<$tP@sQcL5-N_Pay5?|(<|^n?Lk2wJM5_g7f^=B^k>`2u zOBC6ZZ6YS#1)Jj-BW9GA&=(%}AG;T7{3GBgEfdoMFPB@>Dg}lSg_s<^Y-ATjJ-Tx%hWGGtNridWjBa?mh$!y)T6A zH09ku_mc#zjYG(7CXVRMr4o(XY&amvJrw4OK3t z%vbh9lkV^ZGNK;~Sd|9PNsx;p{)G8x^ z1vVEzb-LrQs~c$|laYDCx_2e#+L3}wzPWkezB@;-Br`0^RF{yk4=mf1j`{sbd*IKG ztO{5U+J{AtR%^1;(9*!%zc*>Iqn9$3SUI(>SAunn9_l9|XYcuajb?Tg8tSm!FT8@f zcrAD}3z@A19h8ZXeVTRm71=bU=w`~X^?7si?x3F?U9{}oT`_sxP9+qgm{MOxzjtHi zFTNx76W5eWJ33P&KD$SnUkxMDr6mLBhpy+uPj->c$PHcRnEUM zXz(Mu*-H2Oo3s>wKch!6;M5hM-4qR{uK2(5AsurtKwxS6Oh{tI{+_QX%R+)a{;mvg zMYh$L^!0=D_|zc5Qp7`m@AW$+){E6w@PwpUWA3&2*>MI~)qtX0b9F;pleTbAiSKBw z)|Edk!PU^O2yl%xqA?_?VwDo;TAZjNL!y@<;cAmMe1{=IZc7FGjXJK06&o=?{#LL7 zm4v&5Zg#(a0g4=s+;Fkw!vY+|DvMWx%>WK<7`?N;HfkViFYk76$t?wdDgVlKQ;{op zwYlks)oRfmxi&8b^?|2~Gf048GcPgTmZ~8c_?pLHx8Pr9pA=4mFR(tZbX6V0ceJf)f|c^vO!%d*ybhiz z7h8ckwPxpx*NY!LIg+jG+Uv@-hzzi7e$^HK#$Z>=nmb(r($ELCn|`Hb9M&Vg?9_?h z3d)FyrF1&3LCjCJ(JHMYE_blbD(^TvnD%lwNY(4!AjH(z@5a8(LzqSU4j}FkQ=J*n zoPg<|d`G*A)neJZZH)v;&H~Q5Tu_tHnqZBm|0O-`%H1bykeBmaHn0TvP%-pdrD(h} z->PAO%3@qG)6FkM#(-Le8U3(-87n0UUL|stz4J*JQTZYY9%IHFFIo612!c|yI=KV! zV`$KfsL&!qjdBwTXG#3jlGAP8CHTklv%G&&vUyp+^GlZidH?TEa@elY#?f6biQ%j` zelE{T1cwx*=Xd-Nbt*tpqvhB~wE?)j&pm~<^t_X~lYw@!F<_Q#LF@z$c_b#AeVGQW z`#QYk0pi0R2A2G_hrxQI~^H);5$>syT6{xoXp-L2Ga<=sA*Wv z?deUvqlG>}Bgr_hXW=9j9_#tf&*ow#J@~{JwFCYmx0Xh&YS5)wzl!vJMcFvTJ;A!d zV*N)eD7btW!YOQ)#zp=XBRL~E4i0-l*nBaX?L(lKWXJO6e3P6OE;l!I6|Lyxbu4~i zXc>(tr9rct#yt23p-J!79^hlk<~2Gz*UH`{7t#pnxo|D$%}$&rd&AKBd-t6P{<$91 zYS$$RNwz-xJ#=lPp6qN4vElbZ`gODmnMJSQ_(Ed?Z@gmtXX4To^a?=&v-0OOZ&b&b zvVIVBUyIuwPv-f+8YQ68*BkA`(AL(_Dk)$&xkf7SfE+tSTY)6I29q0kJ*W6yZvob8z0g)4B86pbo(`-LX7C8}K7_CuCZ3mxQAKw#&f7Pz_&f z4!2l(hB+NJ4{Gs<*PnW{ct&1v`$WL=L3vy~lZf3{A?OkzHlgY%s*Viw)I>OObp=*d zo^B{5nTHs~Bp>6Zd&m2B-uCEx%;GX>WTm&Xu-8|uRVULG7&V)GmgZq~{*qs&)0!_6^Cq4}?YMk6pAO_YDWr z7gg6J2)BMTU*38-t0NqTypY}A2&-5{nrjgldrFo&CZ*Q=wex>APic;3^|x&EE)CNO zNs3tw6_GjC%$w@QPn0jX2n?3(V!F>ouaxUQ#<_)@b&nuTd-n_Tb#bX@}GA70Bb;fU&*Ut9d^E@D2NPoxo`?4BV9_pf;=h~M1R><+w z>()RQ$NkU2Yq4wF>4)avqI!$PdFRef)?r#Hfe#Iy*(-s=YRjFpc#g}eCI+VJ&wvu+ zmalKo?{)Ox4sZEtD=k-Og;G7uS4lcOKiAfLxV2J{rKQXb^=Z-ojI)X@w{5kzn*ZIwompX4KWI9&Vq zT<=pI7VCX6H{0edGD&0MEgGTkWaX`myn&yz)fsni_2K?2b@G4!wt!}1Jg#sQGilZP zgwQNt2F>!Ue8?px#L10aD{ODz;T~GqoM=uE%dE@1xy!`z+lPw(t>$iP_axb_N3f3r zf2`x;$E*Bx3x(XvmkDR>zZ(Hbt;S8arCoOB#)bV$YD&JW@sqLlz7Lx}7WMcMOS5RP zw)#(X<+lso@V7^A2j@Lo<6Wzh}z1WdNu( zk7GSClNl+oHAWN}ELnpOoqe8HNJ)oX zWfkaNQqZ^puAzGj zN9w2lQ2nU>;o>l@xx9F)IpA8gxD#|-$eauYjoCE_GrYA4(^J4f>IQE3Qu63=$YKZ+ z{Z!asJS9jLv4MG#nINGZK1Pq9TJI&WD z`WynE?n}~;nfAwOw0qFkS;Nz{l!2|PUK53?@M=Y&$*Rp4o?a@_{BEA2H$7B+-ZU+# zu}8E0rpJ$m%bPU2ZmuqEO~Idzyy-KQ@K{QtNjk(eeM8)3=TWcc(@^+VndtIT7W~(-u}cx11@%HtYIL?l2FzoXCUtxNO=3} zQeB$%S1FwL(OOMSBto-d;5x2OFf$NA#_3LotWWnKW2vX-kg3hoFlDs39-f#SmMNca zZtt#9_3Xx+{)>K7fPaPxO6p{t8$g4XxJjVr4_~+zH=kuuoTPDi-eSt)mYsdJ*Y8f- z1O2PZ&o|QSxi;I_Z)3eaKr3p!1_zTh`25GwqS?0FGHy*4c8C%hFnWF4>=`8)6IRuZL;IWP^1A0&g`DBXzb@+4 zC@TPal_~goWacma+rwOB+hKKr53ZW#F4DL5Y>#KkVM=WPi+i8f4t2#2FVhHNN}ZdR zJRuAF&VwZzv56cthem$-hNXkNwB?m`PmK*&vD#PAhux<36x!<*o8=5>V|CF>n!qyN zt9kZ?uigW+gMka|r>@t^%C_N#*_f1=>*(ZuYbsu9t-A{&WhvxdvCXgBBOy-0&^^44# zQf==9#TFR>I|*Z1DVGTHfOxM%!B75ZeYX*>LJqG>HM@?cBGy$8R;TMSMsJhhrj_!F zYzFdg7q#)(bNK2LBy=w*b1YZ$oD2>2hVfwwmiy*{18p~7C)1O$`6-`*VzoL6wHpR7FMW5aTeFSP9Y_Jb#DHAmg@yW zST)Z|sioYAx+!6xJ&WB$uY9qyqV-W}0@LWzzHB$`%8#|iZ1Y`7Zk{1F*@~GNH(}aL zDn6!T4PYFEHqm&Xuj$8sGIc*H4lyc>pN1XYY6678tyh@wseb-(rX#B zk}2B$ELhKF&Pto3`myhE5Uq5A@vu_%fxdXD;=(FQY!L-n7ZCMkX4zb3CROacG-3Ba zl#iJ(z~V=7!ci10OX+3V^8rzZ=p0K3%bjFw;veX-p(|M(XjKgRR(v1}(BLg~5l-;l zzH2$pw6Nt{QM_CX#;sR{NMvrFx(ni2Z0*tY>ZUPcBqH3)Xk0s`GJdHdVEW?qyBfhd za-lZZCOx%>_d?uoJv{iv8=^DCN=1}BnWJMw=-WDMv2Bs$Msk8Q&b~gggq2b?InY!K zws_%sO0PW|6a^>Y&3SwyzA~FfOv)%H9zXFSek#-FryoLxAIlsiCAneZd?Jjj`H3&( z(3P}rxgvBFoO9Q|Gv--cj*-2-BrbpKY&&4$JHOQD)1FCc$mXzT3#6A+^_z!Ce(HIu zU14B~bK{+$58>lo7x$kjtxQIwehtOY{n}bs(S2ZJGP^CfgK0#5?A$e?oQtlm8xCK>MWp;kK37T-O5MoG#|w^Tq`vW^yjKH4Gy?7%R040}__?V8*1HYJ*86t6sGj2?VNCic3{ zLMxofUaz>P%qE83);0DSEI`9f%Q1mjJD}uW*i&~eXc!y3%v_sriuIPA**9G}xxLkj*C(Ko)g;q#!t=#L!r~XfM zYoCzLPmV`wPmZ}_HU<{5FId@JK*5m|HCNGJKBX~Qh>a*2tdr-NbA1%OLjyBvZEUH{ z^yb9uX(+!>K})__3bI=Ykx|s^@gF>sGxysj`hq|`bC)xzsDI_WiP_4WD#e8UFR8Af zyjB4VXR#3L;^n8kD~u1V9ACfNaSMe^_pA*6m}<`53b01zXF07f6)> zZ}A&gT(7mRHHlKteZdfuh(yDxU8T)imc=8kjWcuMoDsdzaNVuPMeV?0;O44^&OK9& zB$2(#UYB7du;+A5_0EiXI&_y6d|0*#Zwop(T0D@=<=RuP7r-17H~p?_qulz+LQRJs zLSdtb|mkn@TZ4C{3K$W?^-qQY~rbAv~BGD&ev}w>mudL#J%{V)VUMq2+)u&rS7En2N3f&!r8Ud^cxCEbh?uxPr z3BJo!Zy`%0Xo~8EiZka4%jZuq!fi5KM4CugQV^eqsdT2BNjI2F9bUkcvY%pwLo6sU z=K&t9Xx&2Nazi4d#>DNdml8lqSSI_ouWJ}FhYwq2HGQNJLA0w&W`mZ$0YSjrW6(Qs zN`wlDCi?wW1VnF*0XIy8f}DsfQ)8IKt*_xZxGIp?rD54gv}6+_HhR)(2FHbh2(yA% zWRVOFY%cE!kOZNdiXu+s55gLc5Hrvw*NVX?xc&BaU4Ilr?Y|8Lrvj41XV2e6iQWQ3 z*URjq)ug8g<^DN>l7Wp+NAD zbI*dKea4QJb8L*WZ;KilTDj=B8_J$*{0ATXW0Nf4RCT}dFZlCRcyC7-@v(@l)uBQW z-J^`jgWnHQ{=rR;{1EI`uUeU=$*8ySb@=!8Fi5rB-+%`ztiCDV%q2EiuCK589gRcKB^; zy)X8d-|#Fh!WcNHE$>)?{xSJAv!oPZ{po(ytBWWx2ZDoH%;;ZWYQIE=T;2=gHwZSY z$e-0nHfMzru_FU-_tap3UYrL1X7zF23(5aBxbc{=_S|2nYxqqtn)V5P((-@=3{I>& zY-7$5E_Z2w4bJ?hL7G8JMV^du?SF(U7;dgtsrZJyy%&q~ro*WlLr+CL$FdQc<9-)* zTxaLOLhKGmQDS-?0?w>hzww|a3cp|9#?IciHVf}=M0biwUM3Xqo=%qJ!@k=CG~y{J zBFEBK$m~^o>1NOI=(bCW0xZloAW+bu(a*I(t#dsSMgW&hIhEqv;D_mu$dBN@Um6d zxO|&RJ2wImm5#FiIg`tER{E>V_>N4W0q_#p0i=w0WU}FxX>mQx2>7ZO|NY~r6X6Er z8pu8B51sYKAXmrK+RYwLM17KBWD~mS%d<;+c-vJ>xEzE0<+0ybiS%{Nzr(wOFVNgM z%(mahDMULF)+d-}s5{=TDt$H(B3xMB$p4;nh^Y4qb_jafwOW$#4~jnnE?bqHCWLQ=gkSJFl#<{dak@(HfGL!aL=#@2jgb}l z{@&?sxC3HaAK9_dA1;8pK|Ynh8M=-;s+aunTM@R=>9bt*g{9! zG9zg1Nm63!M!L;sWXxqdHaN7L7ndP4$^ZE1+xQx6fmkKwkWBr4mm7dF~k~L(XFS z&sOA`O$Q~|3p9f4)QF}~1iE5togRZ^Ukw7gfcBg_wFheej5_?OVA_`lTZ5*ZIW-N& z4PBZxK3v}CsW`z(yQzi}kND~;0JH|+vp7<&10U}wtibx5bcsqVt*0Bq9gyw#?}Qad zVAjgKsr1L~FFbVe&ECr+<$R+_Y#_nI9gaKdq*f@uKuuT%uX|Ab9Z)_~S7;rkSi>SI zhuS|&EeS>__{d(T8LC5^v)J4{z!Inf8_RpE%NoTzW*0=81MANa9O9J>ui@Xz0x4%t z-Nb|%-Eu%9S|du~c&BuOl&z!5Zg}9w-bpbF^~A`T2G6x~PJ^2)kb7bCskjBxm1QX> zO^0rYjr;bQAZPF-cPN~A9727|Yi?$nz(tKIWbxUl_r)(#%gpR*(4Z^+IQ%_gCMnqU zB}tPg)YC~pS|!{W^YgNvkxJ!q9yNdBvT66SbOq+1xn!k_eYCuBnYkjp;WEJ?TB(=; z&$S67D9mK^4QVnHmb_=yXHY_{Sl2(6f5CB7fD8@3`L;v>w0t@?0hSn4XeSJZwgQ=q7lCAaId{#E3+=;nwH{UJ`{ukx<9ih3&1NJ z^J}6I^s|sbj-Z4RX)pnoepE z-`Szbt2aG&HwUTno$1%Vkt=HR~beSEZ@ur_iZ(7{v*N|U{OihK|vUIgl8_Y_z1$i zpUNv0vE`r6#dg6t?uX(}vQylqH0Edh$Zf+rhF|I$~!-N=fq5K*l|e7cr- zbL9%2YncP$_=%Oumk{K&dmg8L@6Sf)jQ-hJLfPWB&cw<08Hx_JSU>NsIc$cWc{mIK zHh0O=XL)y#)5UjZ8k~n4T%t-FM*p~L7<`-_rxOQ3MAKEX)tw{mVTVTT>@8WpbxMdz0B1 z>$-}PUHQ?Dnr&PS7XQ{nXA)9ADOP*m*?v^A?^wi|p-nXybFh&u9ELkCl=o7kvSo?0 z{)AiJP*&}rfvNtH$V$j?mh=$F#9gFMM4Qz&+VwB2>5P`!;?0~25rFqmZ+c^iFq)C< z3$26!_V9B0taDPzJaM_04{zqO?n4|R!&6VC+qEBC*-r)|6s`qhS`YdaI<4frb~fHk zCjGo#ukW>=O0CkK$=oP}8D;T((jOA_N`3d|5!DPOZur{iqSEN2xF)%C^4XFVOB2NA zM7Yn}wh}tJRLQrd*2E;e7q0*i52&015ISd14~xRb05kZ4>nBCPE!+`m(FU>cgBI4u zz#@&$rr5$lgd#(D|XVJza?u32QrVY zMjh`K!ltG_&A0omnGo8E*n_QH8Kl@QJ#P8|VZB$Xs;aQ}Ak&9YEJ`Xh&4UeVPlKN; z8>*mjF>Nd`eDCrMsO*yDz{t()740}E84!-;Q23iI6a=K_{xsa7y=6wC@#J&VAqMg& zmaAUq(~-uZ__qT&;6mhbzC=u19tih@((dWw&MP87Tl zM2wqn(7|%L8#Nz(cx+Y!mt%yhd{bG{=+zp_E{}AXO|f~A%CR_L{ai``MPzEvh{YJu|(eK!MU2DBl-jF%?!ZTZ@A3r(Qf z3qxhqUkyH%h7H_8iKU46BXa4{v=yd#krwIqtkwM6OM56qJ2@Kq-V#hK!f`Ij{j5yY3C8k^u+RnEM&of|k>{g0dlOt$em0Klu&~rk6z_W)8x4 zM`!p5$NJowAxttc;sj`J*4N5GVC3GKJ+u}~VL2mMu^p{^d>?3g2N9zc7h{)qF_=70 z#+66)mFa0uj2s*8=OK!ZyKqMZ!f$X*uv_H}czob4dpL0t|1n%)>`F^AEuIdFUktN> zx~L6N|za_;m6g8%D_LQv7?yG$wBKZzxr2S=H&@`*&>BMmpBjnJfnXenF?-fVZ< zEd2!?J1^FvMr+=57~oV#syXYRQbz?)VkMw6*`RAT%sbvs?3S}*iN_PkcNUR8YYcd? znJUByck@B|1Oc6{;;y6(j0SyFb$n@5$byfcc7Q|Puwl#duYeJyjpnX3+sQAxqw#e5 z%}E*Q*mdvU%2s!RBn9$ZCNghz@-=UB#vLy7=iZfKy3dJaN*8^FcYE5ndMogB!A-X! z4}VdD&~ZTMcktRP4<)A1z=!U>aDcAd6TmR3GcE~Y-CReULJ-PeQUS&#DF`%*sduDY z`ly{1TxHx&1fS#a5VKvq{f%%T=~;GfhCh~A@YC&eIbbqQ>0&rhfvq{q2p=Maqsw%| z9PBZeN$>V3w?}&4CO!HQ_mcXTZ~fl<$oAhU=Emvg1r zfv3vY#dg+&K3qx53KYupl_|O$6K`44ezPDGdo}&iURV zLFihMK7OxRcOND8@~M)7Vu?^iQVf!8Y_WME5RDxpvS zZjk*W)sUb%@3TJ)8A@M$QD_@bxoqXnPP4fbe!$|ZdM1ck{C2+pRC-Ywlj`)@B)&~gY1i?I+kq@-u< z7-ANNmf165-rc0lL=<10vO3R%Nxk=v?0z~S!X9Tkp6u4Ht2Y;zvmyT(RjvS*&0{we zjV$a3{de&+ssa6XmT>t@&(em(I;w^qvMGQslv z-zx$X^#o{#f~Vt6To3tQne1f>3{wJ_^V8zPX<$nKV{sAYpsz(4gcnRtBTO3*M`Oi; zWw?!)3n&QWZ})(aA1~q7!QS`(Rd}Cp3l1EFZ{SG@HlX~!J)^`46&u-zqJdfdkD3n5 zN*g1H11F#PcM!R>c)|;Is=u=}QIM^sfw|(moyMM=!<6# z4ZcSp{Z}6QHGC-BA9<%~KbYRbIz+m%5K%|wAOCa)fs-TT-{SR=N(~yX;kpZfX9v4 z-oX_W&er&*!)h91%thxab^QD6HRp#s!TZ_+lETBpO4fl^Hr z9Bg_xU%qhk#Y5aN2a1B2U0zd}sQ9vaqpX=J2?3YVU|H^JJjr3kL$ft^b}R0fjAmY8 zz+c>1=ezfLfvRLn9*}T(Wr5F_+wD6M|M3}{;5RY+4uMIX+(QibhCRSJfjAwz5Hw5} z*q`XUj#$%^+;st8GW*+88>zT?EEkhwA8-;D+}AWtm5vlOOD^NQLjk4`_lx&ovXo31 zN!~!spq3DJ#NK?aRMb#+)Uzaba2H0CvqW-G`5RA0i)cZ zJ670`Bl3;(qh=&iG&di_ZN~QL{GOhDk{2oqu?q& z)!2LTS{U)h_HY&B-zt7Sdyy$Q?~*4`@Kqq{7;T?q zBYX>*_dvM)=H&T*s$dPcX`3=OZ=84A(Q@CNBbC}y5*oCDmrf!kSfHJ8fD4>t}qArHKn_moL&q4ASCH8f)s*(-E13 zX3qzInl!rI|L{nIAMC%5{!n9}u#9OOJXjcidpa_FVrQFw^amPVx{Nz^2cHH8QK3XD z=)Q&U$SFu~tSI1jYux-y%-4nCqFyFxy4xysL!osCLROrhWPEUi$uuUPVPu3ATx6@G z)vk}KX0Y=5;kQumyFRL8JH9zw>Qy5nZxH9CJeIzt(3r19M9ekvVmiW!-*wM2(LF&41$JjndK7Q#}>1KG8`%^ZRQTOj+BhCC`mBDTNHS4AR(Or-H6 zp&%*eyS~YRg7EFET$>9*D3B18B6o02{DSUxkSVpjH=$ z$^Q^unu)CkcK4|K#!D?5cdwL>SXdUZ6B-<-PK+^7D{QS4lCd84%1b`O%FzUO z4AQeox`_sMTNZq6ukCwZ0K0y1OerGQJqrP8GH9_T?2N0GTv_7}VeGBHht!_Y=eH$Qd|cYE9BA=*>a6@%Q#S@z%A zuOFmAEB&oU5ZWm$6Orr-oAf2lMIV;ipB!o4{+Uk|e97!1cM#HJJECr^=Gx1TzxPuR zY9@fJQ%Ja#{tOK+Rui~3XKLK!x}PK5brv^#?k`vGv@%Ifv?DY#fa-ts_}qkeAfqeT<8OYD0>fACxiB@cC#*5GGcaEx%<#J6$%NC=xB9I| z(FV=BPS4#YmXm9?lFk(lfiEYLPG8(mf!MEjF%4`;7AWpMrN4q@UlJC!nv;Sp=!U@T zKV#p_Iz|1YrbjhDc{0!qLUnqKw2}THO_d=8#?}U!d803raSUJMH3irLy($BX+|t)Z zOCA{zrptLeiNC)v{z#KMHnh~8_ zY4qgzQvloeFGBG6H7_)mzm=D-Zob0%S^b`ghJK)8xGry=%2s&+Ta-Zo_Xh(?jqCmz zJk^&LzxsQv-Gqq>NXum`2Jyb_)k{eG3ANDMTsHEM!#V+MGZFA7j1|Ok)j_-~s3U^% zkDH?ka00r#5hlH_h8skRV{OT`3t1HmxIoz7CPJMzE+N*vk|o*9WVZ@pEj!= zm!TdYcvrnHwydIrm++vgWTHE1hzXE8Nn-k)JSSkR7v}27nx_$2yDJ>uAS3x7Z|M;3 z4+L&kB$3A2DJ<)*3q(K9S5{lKv~Kfk&FOk?SRi|T*s*(C!+Sioy(cVzOKo)go1M8b z;L1>9wlx_V`{?=H?$G^oxw?uZ!*WHBmXaSd!uBa%Tzdw51pU)v`kxWC;Y62xvQ<7L z=50RZd7?1l!=%(&#wcFcW$V0RERaERW2Ly#5^|sN_Ar+U>tnf zi%vRgZ>XB4{$+W+dV;PIUGi}8*_uANGLY!pFzBb=Oi9#u$!KyzpY6wGFr{l%yAj9! zY@KEGxI^YZfG=qV5LTz==I(W-o{2oD2l0vfe zwIb@0X`w`6hsu?Pu)g`Yuna__4In>^XbO9)$G>Mh%8KO=hgg2QYS>@0FCkKqd;fKQ zO^9<*_bJh11Uyt2eOJ}6Ec;z6q38Az`X{PxU8H$F?X^7%VI=LkPzWRVRfzoxXQ|o) zUc(U4Ur#d;^-9yO=D_cJyouVc^(YSO8~mwgV8N|?RpCo{ZlNjXxi__n^*;I=6;;ob z-nJSI<5^BnttgH(NKl2|a&TT$;DWe}rki9maxg}i$7tGhrEwAmu3$}hB(6}^x4iz1 zsjhwZ=Ya4lz==+=Y$Z6T(Cgtp`2VONoBN!?t@BEMMA}CE{>x$4913p?AG2wJ!*d6! zak>4Es-ti-MK^=xZmnAIbNUQ19Q!eF(#Z+Fi?(P(mGg;lbZ%0eEN}px)Ez&5%I!?($ z_WXVbP@Tj}zO5k%X_8YAKmb9>-J@x@K^Uo?UU6YrF~H$RXr0zh zz*|}SQv0nKgnK|#_ouegf;0RMHLmA9g75fVtnNi_2qPziyvX=z)A*c^u)caczglRW z(nxujGYg^X?HH~jAz!fD#n%6-ki|0Ne4Rw5nII&k0ADyvzhyKh3$T6H1xUNIze_Pi zvca|5x=Z(*ov9eVepcwDSmsxVc%`$o^ss7~W?dh<_cM{$^ZunsYEVWgg@Br={tvir z)aE-To@jh&ycDA~mt+R(0ujA6W+w;REC1(E5!k)QqLmtE?XDf{Mzo8OfM`6oT}z1(NQ>Qz(h*oG~Aeq*bQRgG8gK!>Fv%XH|K-#IpyYbA*P>J$SXOn^fl5cPh7 z{WbKs-V+Rv;c$3F-Dmac^y`6xuWstjf^ky5Ht!n*JK5>cfj}+6>-l64Y$NXEn z2g=}~kYWGfJ&$1vDWQ>8gx-8Zg*gz3xH1cV8w>>bnZ!@GN@{8SYzvhGe`$$^Ak}gw z4J%YZ@AcMHNOo~=v3uxf=o!eAptVvVhdU8q_ zD;+D8hM5MI*qSO_(mR$R0f#wR+T{GSki!XV#7l4tne8as@F1o1NN5I2LacsR~rEP8$f+z>DJ*h&JrdtfYZElxD zQep{T3Y6ETYI9c>*ie?w@ z#i}k^X_0yQIuo%yRrWcu!N)+m{iKh8AikM%W)9+O&+D9{>xJk4L8 zX(n;jb8{lQ5Sxd;`=)2Y+`V~a{wN&NixgCVG@2B0~n8B%VyO?)#wi$zh0))Xa#zc}#^`DPoEA+hS^*JIDQCO)|B z>{GjC6b5$!|CfnhZu3C9+Lnv_+bDtW7b)|e?`^V9)-5LXjqm9EJoy5Wow%L8dz4D$ z!s@)`tx#)5fxRc)n?=d}5h>OK`>*FYJwA@GL$z(`@?Mp?G8nZR3{mJcm?ab%i}jpI zdg0U$dMh&4<7M$yvl5nSf656(MA@@Meg^(eJ69eKb=UrtXpzTAjgUPR8vBx^o>G>{ zK1dBl2#=YDtTU5Tge(yWS$nc?Ls_CRc&f2SmXK{QXp)S+g!EK?XYh9Qdwpc8geCHY;U{1l{UcDKnuV>s}wr}mdbV#Nh ztF@S%raPCNDt7i9zWq1@!5k4HAlkErB3-+qF1)cSc(dE_q96Z!WO5;_{wFPpw0b6l zk?Xwnna+Ir0aGgyP{s;FgBlJBSto+{eIMQX!Wh0b_;xIjFE;+tK3^B>X_=;G8^Ud_ zhs1~1oL?)^0eK?Q;`Hs%)Pgq`mQ4^m_MZ-+Y==6(jfl3ILi(>U`j9~w$@?>ltI$S z%~QV4M=(MNi?d6QS3LZ#k3S0f>J|9K;>$+>!yb4U8rkt4Ue?(BVP6(9fd#b*>@CmL zGQ}0$qSU?(ogBxyd#>#v>r7x;1&X-v1rrUPUl1$)lshr;X0lJa)Z;p|4uQ1Bd0|S> zW5kMu`&gkyOwj|!`+VDFKPkPdWmosIzrQY~9 zUg#b%kh}aD3NzKhIw0<`cyp(6ki|Jt{-M0W-v|93#^YXoyLFl>oAX?GBx+Y1CA#1k z!$!Uq_2p^b&SiMIjU7+tr!VMaV#ctAIYAxXFcom#7)MZXFnUOo{%B?(>T1%SvUE)( z>aQ!Kt!^V9@U?YnPp2MIoYI@DjNVOWwmx~%TDZ)3P7+3TpOD})GesV6cbHr#GWr2N zM>SMQcb9+m1+s=t(8094LT#zNSU2*iS`qyUjMd>OoSIqYAJ!d7PN|LdbTymVlx~%b z;k|Ns+G9lzrqQaooJag5T^TU}99*U(t7s(E#fjVE{T3Qe7AM1$Oq3Q^`G_zRiY7L5 zi>Z_V0%gQ+5t?L%!`PWhe~8k1tVxFcW9+om#*u|?6!CxLnd)k(8BOXpM$}QS#>qaM&G^BDnCiUyogfyyPI~5UoI1Gu z4mA2TRp{_>VHA&e_3Sml#!Tqdxtm6QSxDu+^O$%F+^k?_#&?N0?Dw_Uu#2IgALE1oKuWY8y!flnM*`S@BVdQx4ZzA|q* zL~D>8ymGion>G83CLMTWt9@D60l1J8ruXLO$bpZlIZ<`Ew%ZHgiS!pwKjXHDvWJ*! zNbRZ23*kniEB&Gbt{F*`d7S)8lP|8Cr5R7U@#q?#l6AL{BdvQyv3*rwV@ z9y5wB^Z#B&wk|OU!?wj>n3uh?%er4k?4Czx>J?Zr*>aezT~xv7RnS|>}k*|_im>9NO4&6S%Xfw z8d5IO@EZg1F1^tYoj0hIUM2SQjfk%FCRT-RE*f&W%9TeiSGi^Hx!T{|0zkW@N7ctH zHa*IMR~G8v3sCnuQjxq~vBX8sSx|_QvQU{-dakZ;V1d=M`@Zp28g6<-UDO9F>*ywO z4rM9=BDk|X@|c%9CoGjcl16{}^xORM(&N0G}ZJ z3@e1Odc$MTT>=L~l~P}@AsI#u7hx5`1nz7tyI^3<+*>eMT5-fH=BnC*B4>NK{puCr z7Mx(@c~a~CF?^wJWI?W;OEb%E0shw`BZT4~R)l^E|Eli8Q@sZh>o^j8ljY{E>x9ol zp62y)y)bG`PQiTGQ&zGgx!ryaQH1!6We0)$*f53@eGTH9ORMh05`DwYIxO|$+vEy! z1B$haZL<4SvweT}7c5dtCO+8aQT-;XwkHq0*N|RfU*Z?3nOv5rpK;sXf?1(WO}j;` zmZJ-*-P=j4C4*1o)c$_T*jLXl*BvU7yQ?KjEZ&b4slvz?KhsOb9Cq!m0hC%hfqBbe zdRirarQDn0`(Sg>VQ{rpB))j;JiHu@ELOPy)gM{gdG4Nu9{Q*5$XTajQshOwo+6WT zRK=DXA<{?@NyJatkU}Mpym2e)6fvVJ-Xkn||Dq^TsC3{eCaU?w*B5-MxA1E(|6#`OL{s?5-Wk3s*?DRObxgE@MEe2)7xpY zoS@n-09Scp6tLDMv?Va&g>Xf3@QO)~klX|J%Q5}=Ai}=hQM!+jtNv+|iT-a7lm%Bi z_G-_oPK{c-Z@yQ0lwpG#pQEom2>3W=`p_hMX(oK*Jo;)>O;mT}G|&H~o~284Q%`8mA_&Mj(Wm8J^r&{i|__K@hTAO?dthUZv*^RhIZZ- z*ld*pB>{L`JIeuSXU@3+zLY6r6yz5JMkAUDhsbqP!RzZ3u{V++7`PIZ^q6GH4EtBp zztQ+NyMB%!Y$s`YJm)_BPIMJyf7X}J=q{0Pw!#BFCX+PXEV2r6KEP+WPURR<%Pky1 zR#e2h5vZIY>VZ1ahF@sP(e>j58B3h(Wu_N=5d@LRjbA{BuLtmLK!TX<3Xnmn0jzAp zLnP@&fTF&%;W#uA4?r&$TlXgr;sRXv0k%bO%z5Pie$v$q?_VD<((oF|W8&Ij^KigyTVJbeS$c7G=oH<5$am!seoTV*;hy}`9p3vGqk5Yw z=&aTlXf>ka|AA972S0yyCX?c8H9|FIBQt^|)<{-+ zjll>;RK~60m-WCwpk)r=OCZ3qFhN;KwfVcgrj;RDPT_$lo9+-1zqjlff;BR2P+6;& zE>+s~!-_%g0{@o!!o2u^A349F1tx>i0B-eypexDp&lIXe@5D@;(=BWnB?e z*|+-rX0zqdF$u}jR6BEE_cRXbtQ2jk3!3aKZS)>1w9+%V>?bcdguIdVMp>AV`#1!U z`lHq5Jx3_5d+HGx*Ag(cSz(!Lvr)U2 zFxj>%tWV?P)AFd=YP0U(*=iQJ5twWkH8bJo(eu;i&mrbdcQ#&-w9c><1xi0Y^y9wu zzJ0nho!!||^rhX$&j3!?5jJ?k*A-^&KHiy7;aoU79h%)GBni(&xx;B{`d*1dru$Gi zOGTe0w;G-Iw(3o3cD7dquWLE zn4(Zya;2g0>Yw8sx6G){a0ajWK`wUYVU)dpWIlcv`zN|0<^b%)fcU8>f)j}1#7j3! zqRJDDr)=S6m2*Aca@3aIzOAHQzK(ts6N88V1Yp21QF}dTu78ESA+alBmX0|O-}cu4 zyyFIbai${)Yu&=~muNe+I z>S#V6WM;Rre2#b=LVXLM^ga)6EWWC35*NX=f$4P>7b>fjhE25|UVa;I&90}ZPB4IZ zY4%Vl3kM7P1SkAmsM`}=m<3J7MHm4kP zfW)}c&vXNR{Q;0X3H6{f)!P zYzwVj>|fKx76-{w(60itKH`XIBw!E z&5L2YcF)o|4CehIj+Q;=nfx{HsS08o7q-sohA5~x2=d)L8tgB&