diff --git a/deployments/common/crds/k8s.nginx.org_globalconfigurations.yaml b/deployments/common/crds/k8s.nginx.org_globalconfigurations.yaml index d4448bfbc9..b0dc371fd7 100644 --- a/deployments/common/crds/k8s.nginx.org_globalconfigurations.yaml +++ b/deployments/common/crds/k8s.nginx.org_globalconfigurations.yaml @@ -45,5 +45,7 @@ spec: type: integer protocol: type: string + ssl: + type: boolean served: true storage: true diff --git a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml index f0095f4dd6..189cce4f6e 100644 --- a/deployments/common/crds/k8s.nginx.org_virtualservers.yaml +++ b/deployments/common/crds/k8s.nginx.org_virtualservers.yaml @@ -97,6 +97,14 @@ spec: internalRoute: description: InternalRoute allows for the configuration of internal routing. type: boolean + listener: + description: Listener references a custom http and/or https listener defined in GlobalConfiguration. + type: object + properties: + http: + type: string + https: + type: string policies: type: array items: diff --git a/deployments/helm-chart/crds/k8s.nginx.org_globalconfigurations.yaml b/deployments/helm-chart/crds/k8s.nginx.org_globalconfigurations.yaml index d4448bfbc9..b0dc371fd7 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_globalconfigurations.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_globalconfigurations.yaml @@ -45,5 +45,7 @@ spec: type: integer protocol: type: string + ssl: + type: boolean served: true storage: true diff --git a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml index f0095f4dd6..189cce4f6e 100644 --- a/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml +++ b/deployments/helm-chart/crds/k8s.nginx.org_virtualservers.yaml @@ -97,6 +97,14 @@ spec: internalRoute: description: InternalRoute allows for the configuration of internal routing. type: boolean + listener: + description: Listener references a custom http and/or https listener defined in GlobalConfiguration. + type: object + properties: + http: + type: string + https: + type: string policies: type: array items: diff --git a/examples/custom-resources/custom-listeners/README.md b/examples/custom-resources/custom-listeners/README.md new file mode 100644 index 0000000000..b68bcbe03a --- /dev/null +++ b/examples/custom-resources/custom-listeners/README.md @@ -0,0 +1,181 @@ +# Custom HTTP Listeners + +In this example, we will configure a VirtualServer resource with custom HTTP listeners. +This will allow HTTP and/or HTTPs based requests to be made on non-default ports. + +## Prerequisites + +1. Follow the [installation](https://docs.nginx.com/nginx-ingress-controller/installation/installation-with-manifests/) + instructions to deploy the Ingress Controller with custom resources enabled. +2. Ensure the Ingress Controller is configured with the `-global-configuration` argument: + ```console + args: + - -global-configuration=$(POD_NAMESPACE)/nginx-configuration + ``` + +3. Save the public IP address of the Ingress Controller into a shell variable: + ```console + IC_IP=XXX.YYY.ZZZ.III + ``` + +4. If you have a NodePort or Loadbalancer service deployed, ensure they are updated to include the custom listener ports. +Example YAML for a LoadBalancer: + ```yaml + apiVersion: v1 + kind: Service + metadata: + name: nginx-ingress + namespace: nginx-ingress + spec: + type: LoadBalancer + ports: + - port: 8083 + targetPort: 8083 + protocol: TCP + name: http-8083 + - port: 8443 + targetPort: 8443 + protocol: TCP + name: https-8443 + selector: + app: nginx-ingress + ``` +## Step 1 - Deploy the GlobalConfiguration resource +Similar to how listeners are configured in our [basic-tcp-udp](../../examples/custom-resource/basic-tcp-udp) examples, +here we deploy a GlobalConfiguration resource with the listeners we want to use in our VirtualServer. + ```yaml + apiVersion: k8s.nginx.org/v1alpha1 + kind: GlobalConfiguration + metadata: + name: nginx-configuration + namespace: nginx-ingress + spec: + listeners: + - name: http-8083 + port: 8083 + protocol: HTTP + - name: https-8443 + port: 8443 + protocol: HTTP + ssl: true + ``` + + ```console + kubectl create -f global-configuration.yaml + ``` + +## Step 2 - Save the custom port numbers +Save the custom HTTP and/or HTTPS ports into a shell variables for later: + + ```console + IC_HTTP_PORT=8083 + IC_HTTPS_PORT=8443 + ``` + +## Step 3 - Deploy the Cafe Application + +Create the coffee and the tea deployments and services: + + ```console + kubectl create -f cafe.yaml + ``` + +## Step 4 - Deploy the VirtualServer with custom listeners +The VirtualServer in this example is set to use the listeners defined in the GlobalConfiguration resource +that was deployed in Step 1. Below is the yaml of this example VirtualServer: + + ```yaml + apiVersion: k8s.nginx.org/v1 + kind: VirtualServer + metadata: + name: cafe + spec: + listener: + http: http-8083 + https: https-8443 + host: cafe.example.com + tls: + secret: cafe-secret + upstreams: + - name: tea + service: tea-svc + port: 80 + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + action: + pass: tea + - path: /coffee + action: + pass: coffee + ``` + +1. Create the secret with the TLS certificate and key: + + ```console + kubectl create -f cafe-secret.yaml + ``` + +2. Create the VirtualServer resource: + + ```console + kubectl create -f cafe-virtual-server.yaml + ``` + +## Step 5 - Test the Configuration + +1. Check that the configuration has been successfully applied by inspecting the events of the VirtualServer: + + ```console + kubectl describe virtualserver cafe + ``` + + Below you will see the events as well as the new `Listeners` field + ```console + . . . + Spec: + Host: cafe.example.com + Listener: + Http: http-8083 + Https: https-8443 + . . . + Routes: + . . . + Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal AddedOrUpdated 7s nginx-ingress-controller Configuration for default/cafe was added or updated + ``` + +2. Since the deployed VirtualServer is using ports `8083` and 8443` in this example. you must explicitly specify these ports +when sending requests to the endpoints of this VirtualServer: + + For `/coffee` on `8443`: + + ```console + curl -k https://cafe.example.com:8443/coffee + ``` + + ```text + Server address: 10.32.0.40:8080 + Server name: coffee-7dd75bc79b-qmhmv + ... + URI: /coffee + ... + ``` + + For `/coffee` on `8083`: + + ```console + curl -k https://cafe.example.com:8083/coffee + ``` + + ```text + Server address: 10.32.0.40:8080 + Server name: coffee-7dd75bc79b-qmhmv + ... + URI: /coffee + ... + ``` diff --git a/examples/custom-resources/custom-listeners/cafe-secret.yaml b/examples/custom-resources/custom-listeners/cafe-secret.yaml new file mode 100644 index 0000000000..8f9fd84855 --- /dev/null +++ b/examples/custom-resources/custom-listeners/cafe-secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cafe-secret +type: kubernetes.io/tls +data: + tls.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURMakNDQWhZQ0NRREFPRjl0THNhWFdqQU5CZ2txaGtpRzl3MEJBUXNGQURCYU1Rc3dDUVlEVlFRR0V3SlYKVXpFTE1Ba0dBMVVFQ0F3Q1EwRXhJVEFmQmdOVkJBb01HRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MApaREViTUJrR0ExVUVBd3dTWTJGbVpTNWxlR0Z0Y0d4bExtTnZiU0FnTUI0WERURTRNRGt4TWpFMk1UVXpOVm9YCkRUSXpNRGt4TVRFMk1UVXpOVm93V0RFTE1Ba0dBMVVFQmhNQ1ZWTXhDekFKQmdOVkJBZ01Ba05CTVNFd0h3WUQKVlFRS0RCaEpiblJsY201bGRDQlhhV1JuYVhSeklGQjBlU0JNZEdReEdUQVhCZ05WQkFNTUVHTmhabVV1WlhoaApiWEJzWlM1amIyMHdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDcDZLbjdzeTgxCnAwanVKL2N5ayt2Q0FtbHNmanRGTTJtdVpOSzBLdGVjcUcyZmpXUWI1NXhRMVlGQTJYT1N3SEFZdlNkd0kyaloKcnVXOHFYWENMMnJiNENaQ0Z4d3BWRUNyY3hkam0zdGVWaVJYVnNZSW1tSkhQUFN5UWdwaW9iczl4N0RsTGM2SQpCQTBaalVPeWwwUHFHOVNKZXhNVjczV0lJYTVyRFZTRjJyNGtTa2JBajREY2o3TFhlRmxWWEgySTVYd1hDcHRDCm42N0pDZzQyZitrOHdnemNSVnA4WFprWldaVmp3cTlSVUtEWG1GQjJZeU4xWEVXZFowZXdSdUtZVUpsc202OTIKc2tPcktRajB2a29QbjQxRUUvK1RhVkVwcUxUUm9VWTNyemc3RGtkemZkQml6Rk8yZHNQTkZ4MkNXMGpYa05MdgpLbzI1Q1pyT2hYQUhBZ01CQUFFd0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFLSEZDY3lPalp2b0hzd1VCTWRMClJkSEliMzgzcFdGeW5acS9MdVVvdnNWQTU4QjBDZzdCRWZ5NXZXVlZycTVSSWt2NGxaODFOMjl4MjFkMUpINnIKalNuUXgrRFhDTy9USkVWNWxTQ1VwSUd6RVVZYVVQZ1J5anNNL05VZENKOHVIVmhaSitTNkZBK0NuT0Q5cm4yaQpaQmVQQ0k1ckh3RVh3bm5sOHl3aWozdnZRNXpISXV5QmdsV3IvUXl1aTlmalBwd1dVdlVtNG52NVNNRzl6Q1Y3ClBwdXd2dWF0cWpPMTIwOEJqZkUvY1pISWc4SHc5bXZXOXg5QytJUU1JTURFN2IvZzZPY0s3TEdUTHdsRnh2QTgKN1dqRWVxdW5heUlwaE1oS1JYVmYxTjM0OWVOOThFejM4Zk9USFRQYmRKakZBL1BjQytHeW1lK2lHdDVPUWRGaAp5UkU9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBcWVpcCs3TXZOYWRJN2lmM01wUHJ3Z0pwYkg0N1JUTnBybVRTdENyWG5LaHRuNDFrCkcrZWNVTldCUU5semtzQndHTDBuY0NObzJhN2x2S2wxd2k5cTIrQW1RaGNjS1ZSQXEzTVhZNXQ3WGxZa1YxYkcKQ0pwaVJ6ejBza0lLWXFHN1BjZXc1UzNPaUFRTkdZMURzcGRENmh2VWlYc1RGZTkxaUNHdWF3MVVoZHErSkVwRwp3SStBM0kreTEzaFpWVng5aU9WOEZ3cWJRcCt1eVFvT05uL3BQTUlNM0VWYWZGMlpHVm1WWThLdlVWQ2cxNWhRCmRtTWpkVnhGbldkSHNFYmltRkNaYkp1dmRySkRxeWtJOUw1S0Q1K05SQlAvazJsUkthaTAwYUZHTjY4NE93NUgKYzMzUVlzeFR0bmJEelJjZGdsdEkxNURTN3lxTnVRbWF6b1Z3QndJREFRQUJBb0lCQVFDUFNkU1luUXRTUHlxbApGZlZGcFRPc29PWVJoZjhzSStpYkZ4SU91UmF1V2VoaEp4ZG01Uk9ScEF6bUNMeUw1VmhqdEptZTIyM2dMcncyCk45OUVqVUtiL1ZPbVp1RHNCYzZvQ0Y2UU5SNThkejhjbk9SVGV3Y290c0pSMXBuMWhobG5SNUhxSkpCSmFzazEKWkVuVVFmY1hackw5NGxvOUpIM0UrVXFqbzFGRnM4eHhFOHdvUEJxalpzVjdwUlVaZ0MzTGh4bndMU0V4eUZvNApjeGI5U09HNU9tQUpvelN0Rm9RMkdKT2VzOHJKNXFmZHZ5dGdnOXhiTGFRTC94MGtwUTYyQm9GTUJEZHFPZVBXCktmUDV6WjYvMDcvdnBqNDh5QTFRMzJQem9idWJzQkxkM0tjbjMyamZtMUU3cHJ0V2wrSmVPRmlPem5CUUZKYk4KNHFQVlJ6NWhBb0dCQU50V3l4aE5DU0x1NFArWGdLeWNrbGpKNkY1NjY4Zk5qNUN6Z0ZScUowOXpuMFRsc05ybwpGVExaY3hEcW5SM0hQWU00MkpFUmgySi9xREZaeW5SUW8zY2czb2VpdlVkQlZHWTgrRkkxVzBxZHViL0w5K3l1CmVkT1pUUTVYbUdHcDZyNmpleHltY0ppbS9Pc0IzWm5ZT3BPcmxEN1NQbUJ2ek5MazRNRjZneGJYQW9HQkFNWk8KMHA2SGJCbWNQMHRqRlhmY0tFNzdJbUxtMHNBRzR1SG9VeDBlUGovMnFyblRuT0JCTkU0TXZnRHVUSnp5K2NhVQprOFJxbWRIQ2JIelRlNmZ6WXEvOWl0OHNaNzdLVk4xcWtiSWN1YytSVHhBOW5OaDFUanNSbmU3NFowajFGQ0xrCmhIY3FIMHJpN1BZU0tIVEU4RnZGQ3haWWRidUI4NENtWmlodnhicFJBb0dBSWJqcWFNWVBUWXVrbENkYTVTNzkKWVNGSjFKelplMUtqYS8vdER3MXpGY2dWQ0thMzFqQXdjaXowZi9sU1JxM0hTMUdHR21lemhQVlRpcUxmZVpxYwpSMGlLYmhnYk9jVlZrSkozSzB5QXlLd1BUdW14S0haNnpJbVpTMGMwYW0rUlk5WUdxNVQ3WXJ6cHpjZnZwaU9VCmZmZTNSeUZUN2NmQ21mb09oREN0enVrQ2dZQjMwb0xDMVJMRk9ycW40M3ZDUzUxemM1em9ZNDR1QnpzcHd3WU4KVHd2UC9FeFdNZjNWSnJEakJDSCtULzZzeXNlUGJKRUltbHpNK0l3eXRGcEFOZmlJWEV0LzQ4WGY2ME54OGdXTQp1SHl4Wlp4L05LdER3MFY4dlgxUE9ucTJBNWVpS2ErOGpSQVJZS0pMWU5kZkR1d29seHZHNmJaaGtQaS80RXRUCjNZMThzUUtCZ0h0S2JrKzdsTkpWZXN3WEU1Y1VHNkVEVXNEZS8yVWE3ZlhwN0ZjanFCRW9hcDFMU3crNlRYcDAKWmdybUtFOEFSek00NytFSkhVdmlpcS9udXBFMTVnMGtKVzNzeWhwVTl6WkxPN2x0QjBLSWtPOVpSY21Vam84UQpjcExsSE1BcWJMSjhXWUdKQ2toaVd4eWFsNmhZVHlXWTRjVmtDMHh0VGwvaFVFOUllTktvCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg== diff --git a/examples/custom-resources/custom-listeners/cafe-virtual-server.yaml b/examples/custom-resources/custom-listeners/cafe-virtual-server.yaml new file mode 100644 index 0000000000..aef5fde037 --- /dev/null +++ b/examples/custom-resources/custom-listeners/cafe-virtual-server.yaml @@ -0,0 +1,25 @@ +apiVersion: k8s.nginx.org/v1 +kind: VirtualServer +metadata: + name: cafe +spec: + listener: + http: http-8083 + https: https-8443 + host: cafe.example.com + tls: + secret: cafe-secret + upstreams: + - name: tea + service: tea-svc + port: 80 + - name: coffee + service: coffee-svc + port: 80 + routes: + - path: /tea + action: + pass: tea + - path: /coffee + action: + pass: coffee diff --git a/examples/custom-resources/custom-listeners/cafe.yaml b/examples/custom-resources/custom-listeners/cafe.yaml new file mode 100644 index 0000000000..f049e8bf29 --- /dev/null +++ b/examples/custom-resources/custom-listeners/cafe.yaml @@ -0,0 +1,65 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coffee +spec: + replicas: 2 + selector: + matchLabels: + app: coffee + template: + metadata: + labels: + app: coffee + spec: + containers: + - name: coffee + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: coffee-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: coffee +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: tea +spec: + replicas: 1 + selector: + matchLabels: + app: tea + template: + metadata: + labels: + app: tea + spec: + containers: + - name: tea + image: nginxdemos/nginx-hello:plain-text + ports: + - containerPort: 8080 +--- +apiVersion: v1 +kind: Service +metadata: + name: tea-svc +spec: + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + app: tea diff --git a/examples/custom-resources/custom-listeners/global-configuration.yaml b/examples/custom-resources/custom-listeners/global-configuration.yaml new file mode 100644 index 0000000000..1cab675af6 --- /dev/null +++ b/examples/custom-resources/custom-listeners/global-configuration.yaml @@ -0,0 +1,14 @@ +apiVersion: k8s.nginx.org/v1alpha1 +kind: GlobalConfiguration +metadata: + name: nginx-configuration + namespace: nginx-ingress +spec: + listeners: + - name: http-8083 + port: 8083 + protocol: HTTP + - name: https-8443 + port: 8443 + protocol: HTTP + ssl: true diff --git a/internal/configs/configurator.go b/internal/configs/configurator.go index e0e7ab37e5..f902637e24 100644 --- a/internal/configs/configurator.go +++ b/internal/configs/configurator.go @@ -1241,6 +1241,30 @@ func (cnf *Configurator) UpdateConfig(cfgParams *ConfigParams, resources Extende return allWarnings, nil } +// UpdateVirtualServers updates VirtualServers. +func (cnf *Configurator) UpdateVirtualServers(updatedVSExes []*VirtualServerEx, deletedKeys []string) []error { + var errList []error + for _, vsEx := range updatedVSExes { + _, err := cnf.addOrUpdateVirtualServer(vsEx) + if err != nil { + errList = append(errList, fmt.Errorf("error adding or updating VirtualServer %v/%v: %w", vsEx.VirtualServer.Namespace, vsEx.VirtualServer.Name, err)) + } + } + + for _, key := range deletedKeys { + err := cnf.DeleteVirtualServer(key, true) + if err != nil { + errList = append(errList, fmt.Errorf("error when removing VirtualServer %v: %w", key, err)) + } + } + + if err := cnf.reload(nginx.ReloadForOtherUpdate); err != nil { + errList = append(errList, fmt.Errorf("error when updating VirtualServer: %w", err)) + } + + return errList +} + // UpdateTransportServers updates TransportServers. func (cnf *Configurator) UpdateTransportServers(updatedTSExes []*TransportServerEx, deletedKeys []string) []error { var errList []error diff --git a/internal/configs/version2/http.go b/internal/configs/version2/http.go index daa83f9dd9..12bbf85995 100644 --- a/internal/configs/version2/http.go +++ b/internal/configs/version2/http.go @@ -50,6 +50,9 @@ type UpstreamServer struct { type Server struct { ServerName string StatusZone string + CustomListeners bool + HTTPPort int + HTTPSPort int ProxyProtocol bool SSL *SSL ServerTokens string diff --git a/internal/configs/version2/nginx-plus.virtualserver.tmpl b/internal/configs/version2/nginx-plus.virtualserver.tmpl index f0a69c365d..9e836814d8 100644 --- a/internal/configs/version2/nginx-plus.virtualserver.tmpl +++ b/internal/configs/version2/nginx-plus.virtualserver.tmpl @@ -65,8 +65,15 @@ proxy_cache_path /var/cache/nginx/jwks_uri_{{$s.VSName}} levels=1 keys_zone=jwks server { {{ if $s.Gunzip }}gunzip on;{{end}} + {{ if not $s.CustomListeners }} listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; {{ if not $s.DisableIPV6 }}listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ else }} + {{ if (gt $s.HTTPPort 0) }} + listen {{ $s.HTTPPort }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + {{ if not $s.DisableIPV6 }}listen [::]:{{ $s.HTTPPort }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ end }} + {{ end }} server_name {{ $s.ServerName }}; status_zone {{ $s.StatusZone }}; @@ -98,8 +105,15 @@ server { set_real_ip_from unix:; real_ip_header proxy_protocol; {{ else }} + {{ if not $s.CustomListeners }} listen 443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; {{ if not $s.DisableIPV6 }}listen [::]:443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ else }} + {{ if (gt $s.HTTPSPort 0) }} + listen {{ $s.HTTPSPort }} ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + {{ if not $s.DisableIPV6 }}listen [::]:{{ $s.HTTPSPort }} ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ end }} + {{ end }} {{ end }} {{ if $ssl.RejectHandshake }} diff --git a/internal/configs/version2/nginx.virtualserver.tmpl b/internal/configs/version2/nginx.virtualserver.tmpl index 2f02717bf7..66e550b464 100644 --- a/internal/configs/version2/nginx.virtualserver.tmpl +++ b/internal/configs/version2/nginx.virtualserver.tmpl @@ -42,8 +42,15 @@ limit_req_zone {{ $z.Key }} zone={{ $z.ZoneName }}:{{ $z.ZoneSize }} rate={{ $z. {{ $s := .Server }} server { {{ if $s.Gunzip }}gunzip on;{{end}} + {{ if not $s.CustomListeners }} listen 80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; {{ if not $s.DisableIPV6 }}listen [::]:80{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ else }} + {{ if (gt $s.HTTPPort 0) }} + listen {{ $s.HTTPPort }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + {{ if not $s.DisableIPV6 }}listen [::]:{{ $s.HTTPPort }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ end }} + {{ end }} server_name {{ $s.ServerName }}; @@ -58,8 +65,15 @@ server { set_real_ip_from unix:; real_ip_header proxy_protocol; {{ else }} + {{ if not $s.CustomListeners }} listen 443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; {{ if not $s.DisableIPV6 }}listen [::]:443 ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ else }} + {{ if (gt $s.HTTPSPort 0) }} + listen {{ $s.HTTPSPort }} ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }}; + {{ if not $s.DisableIPV6 }}listen [::]:{{ $s.HTTPSPort }} ssl{{ if $ssl.HTTP2 }} http2{{ end }}{{ if $s.ProxyProtocol }} proxy_protocol{{ end }};{{ end }} + {{ end }} + {{ end }} {{ end }} {{ if $ssl.RejectHandshake }} diff --git a/internal/configs/version2/templates_test.go b/internal/configs/version2/templates_test.go index 6adbb716df..98b4d9a780 100644 --- a/internal/configs/version2/templates_test.go +++ b/internal/configs/version2/templates_test.go @@ -89,6 +89,83 @@ func TestExecuteVirtualServerTemplate_RendersTemplateWithSessionCookieSameSite(t t.Log(string(got)) } +func TestExecuteVirtualServerTemplate_RendersTemplateWithCustomListener(t *testing.T) { + t.Parallel() + executor := newTmplExecutorNGINXPlus(t) + got, err := executor.ExecuteVirtualServerTemplate(&virtualServerCfgWithCustomListener) + if err != nil { + t.Error(err) + } + wantStrings := []string{ + "listen 8082", + "listen [::]:8082", + "listen 8443 ssl", + "listen [::]:8443 ssl", + } + for _, want := range wantStrings { + if !bytes.Contains(got, []byte(want)) { + t.Errorf("want `%s` in generated template", want) + } + } + t.Log(string(got)) +} + +func TestExecuteVirtualServerTemplate_RendersTemplateWithCustomListenerHTTPOnly(t *testing.T) { + t.Parallel() + executor := newTmplExecutorNGINXPlus(t) + got, err := executor.ExecuteVirtualServerTemplate(&virtualServerCfgWithCustomListenerHTTPOnly) + if err != nil { + t.Error(err) + } + wantStrings := []string{ + "listen 8082", + "listen [::]:8082", + } + unwantStrings := []string{ + "listen 8443 ssl", + "listen [::]:8443 ssl", + } + for _, want := range wantStrings { + if !bytes.Contains(got, []byte(want)) { + t.Errorf("want `%s` in generated template", want) + } + } + for _, want := range unwantStrings { + if bytes.Contains(got, []byte(want)) { + t.Errorf("unwant `%s` in generated template", want) + } + } + t.Log(string(got)) +} + +func TestExecuteVirtualServerTemplate_RendersTemplateWithCustomListenerHTTPSOnly(t *testing.T) { + t.Parallel() + executor := newTmplExecutorNGINXPlus(t) + got, err := executor.ExecuteVirtualServerTemplate(&virtualServerCfgWithCustomListenerHTTPSOnly) + if err != nil { + t.Error(err) + } + wantStrings := []string{ + "listen 8443 ssl", + "listen [::]:8443 ssl", + } + unwantStrings := []string{ + "listen 8082", + "listen [::]:8082", + } + for _, want := range wantStrings { + if !bytes.Contains(got, []byte(want)) { + t.Errorf("want `%s` in generated template", want) + } + } + for _, want := range unwantStrings { + if bytes.Contains(got, []byte(want)) { + t.Errorf("want no `%s` in generated template", want) + } + } + t.Log(string(got)) +} + func TestVirtualServerForNginxPlusWithWAFApBundle(t *testing.T) { t.Parallel() executor := newTmplExecutorNGINXPlus(t) @@ -2546,6 +2623,1050 @@ var ( }, } + virtualServerCfgWithCustomListener = VirtualServerConfig{ + LimitReqZones: []LimitReqZone{ + { + ZoneName: "pol_rl_test_test_test", Rate: "10r/s", ZoneSize: "10m", Key: "$url", + }, + }, + Upstreams: []Upstream{ + { + Name: "test-upstream", + Servers: []UpstreamServer{ + { + Address: "10.0.0.20:8001", + }, + }, + LBMethod: "random", + Keepalive: 32, + MaxFails: 4, + FailTimeout: "10s", + MaxConns: 31, + SlowStart: "10s", + UpstreamZoneSize: "256k", + Queue: &Queue{Size: 10, Timeout: "60s"}, + SessionCookie: &SessionCookie{Enable: true, Name: "test", Path: "/tea", Expires: "25s"}, + NTLM: true, + }, + { + Name: "coffee-v1", + Servers: []UpstreamServer{ + { + Address: "10.0.0.31:8001", + }, + }, + MaxFails: 8, + FailTimeout: "15s", + MaxConns: 2, + UpstreamZoneSize: "256k", + }, + { + Name: "coffee-v2", + Servers: []UpstreamServer{ + { + Address: "10.0.0.32:8001", + }, + }, + MaxFails: 12, + FailTimeout: "20s", + MaxConns: 4, + UpstreamZoneSize: "256k", + }, + }, + SplitClients: []SplitClient{ + { + Source: "$request_id", + Variable: "$split_0", + Distributions: []Distribution{ + { + Weight: "50%", + Value: "@loc0", + }, + { + Weight: "50%", + Value: "@loc1", + }, + }, + }, + }, + Maps: []Map{ + { + Source: "$match_0_0", + Variable: "$match", + Parameters: []Parameter{ + { + Value: "~^1", + Result: "@match_loc_0", + }, + { + Value: "default", + Result: "@match_loc_default", + }, + }, + }, + { + Source: "$http_x_version", + Variable: "$match_0_0", + Parameters: []Parameter{ + { + Value: "v2", + Result: "1", + }, + { + Value: "default", + Result: "0", + }, + }, + }, + }, + HTTPSnippets: []string{"# HTTP snippet"}, + Server: Server{ + ServerName: "example.com", + StatusZone: "example.com", + ProxyProtocol: true, + SSL: &SSL{ + HTTP2: true, + Certificate: "cafe-secret.pem", + CertificateKey: "cafe-secret.pem", + }, + TLSRedirect: &TLSRedirect{ + BasedOn: "$scheme", + Code: 301, + }, + CustomListeners: true, + HTTPPort: 8082, + HTTPSPort: 8443, + ServerTokens: "off", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Allow: []string{"127.0.0.1"}, + Deny: []string{"127.0.0.1"}, + LimitReqs: []LimitReq{ + { + ZoneName: "pol_rl_test_test_test", + Delay: 10, + Burst: 5, + }, + }, + LimitReqOptions: LimitReqOptions{ + LogLevel: "error", + RejectCode: 503, + }, + JWTAuth: &JWTAuth{ + Realm: "My Api", + Secret: "jwk-secret", + }, + IngressMTLS: &IngressMTLS{ + ClientCert: "ingress-mtls-secret", + VerifyClient: "on", + VerifyDepth: 2, + }, + WAF: &WAF{ + ApPolicy: "/etc/nginx/waf/nac-policies/default-dataguard-alarm", + ApSecurityLogEnable: true, + ApLogConf: []string{"/etc/nginx/waf/nac-logconfs/default-logconf"}, + }, + Snippets: []string{"# server snippet"}, + InternalRedirectLocations: []InternalRedirectLocation{ + { + Path: "/split", + Destination: "@split_0", + }, + { + Path: "/coffee", + Destination: "@match", + }, + }, + HealthChecks: []HealthCheck{ + { + Name: "coffee", + URI: "/", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://coffee-v2", + Mandatory: true, + Persistent: true, + }, + { + Name: "tea", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://tea-v2", + GRPCPass: "grpc://tea-v3", + GRPCStatus: createPointerFromInt(12), + GRPCService: "tea-servicev2", + }, + }, + Locations: []Location{ + { + Path: "/", + Snippets: []string{"# location snippet"}, + Allow: []string{"127.0.0.1"}, + Deny: []string{"127.0.0.1"}, + LimitReqs: []LimitReq{ + { + ZoneName: "loc_pol_rl_test_test_test", + }, + }, + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyBuffering: true, + ProxyBuffers: "8 4k", + ProxyBufferSize: "4k", + ProxyMaxTempFileSize: "1024m", + ProxyPass: "http://test-upstream", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + Internal: true, + ProxyPassRequestHeaders: false, + ProxyPassHeaders: []string{"Host"}, + ProxyPassRewrite: "$request_uri", + ProxyHideHeaders: []string{"Header"}, + ProxyIgnoreHeaders: "Cache", + Rewrites: []string{"$request_uri $request_uri", "$request_uri $request_uri"}, + AddHeaders: []AddHeader{ + { + Header: Header{ + Name: "Header-Name", + Value: "Header Value", + }, + Always: true, + }, + }, + EgressMTLS: &EgressMTLS{ + Certificate: "egress-mtls-secret.pem", + CertificateKey: "egress-mtls-secret.pem", + VerifyServer: true, + VerifyDepth: 1, + Ciphers: "DEFAULT", + Protocols: "TLSv1.3", + TrustedCert: "trusted-cert.pem", + SessionReuse: true, + ServerName: true, + }, + }, + { + Path: "@loc0", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + ProxyInterceptErrors: true, + ErrorPages: []ErrorPage{ + { + Name: "@error_page_1", + Codes: "400 500", + ResponseCode: 200, + }, + { + Name: "@error_page_2", + Codes: "500", + ResponseCode: 0, + }, + }, + }, + { + Path: "@loc1", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "@loc2", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + GRPCPass: "grpc://coffee-v3", + }, + { + Path: "@match_loc_0", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "@match_loc_default", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "/return", + ProxyInterceptErrors: true, + ErrorPages: []ErrorPage{ + { + Name: "@return_0", + Codes: "418", + ResponseCode: 200, + }, + }, + InternalProxyPass: "http://unix:/var/lib/nginx/nginx-418-server.sock", + }, + }, + ErrorPageLocations: []ErrorPageLocation{ + { + Name: "@vs_cafe_cafe_vsr_tea_tea_tea__tea_error_page_0", + DefaultType: "application/json", + Return: &Return{ + Code: 200, + Text: "Hello World", + }, + Headers: nil, + }, + { + Name: "@vs_cafe_cafe_vsr_tea_tea_tea__tea_error_page_1", + DefaultType: "", + Return: &Return{ + Code: 200, + Text: "Hello World", + }, + Headers: []Header{ + { + Name: "Set-Cookie", + Value: "cookie1=test", + }, + { + Name: "Set-Cookie", + Value: "cookie2=test; Secure", + }, + }, + }, + }, + ReturnLocations: []ReturnLocation{ + { + Name: "@return_0", + DefaultType: "text/html", + Return: Return{ + Code: 200, + Text: "Hello!", + }, + }, + }, + }, + } + + virtualServerCfgWithCustomListenerHTTPOnly = VirtualServerConfig{ + LimitReqZones: []LimitReqZone{ + { + ZoneName: "pol_rl_test_test_test", Rate: "10r/s", ZoneSize: "10m", Key: "$url", + }, + }, + Upstreams: []Upstream{ + { + Name: "test-upstream", + Servers: []UpstreamServer{ + { + Address: "10.0.0.20:8001", + }, + }, + LBMethod: "random", + Keepalive: 32, + MaxFails: 4, + FailTimeout: "10s", + MaxConns: 31, + SlowStart: "10s", + UpstreamZoneSize: "256k", + Queue: &Queue{Size: 10, Timeout: "60s"}, + SessionCookie: &SessionCookie{Enable: true, Name: "test", Path: "/tea", Expires: "25s"}, + NTLM: true, + }, + { + Name: "coffee-v1", + Servers: []UpstreamServer{ + { + Address: "10.0.0.31:8001", + }, + }, + MaxFails: 8, + FailTimeout: "15s", + MaxConns: 2, + UpstreamZoneSize: "256k", + }, + { + Name: "coffee-v2", + Servers: []UpstreamServer{ + { + Address: "10.0.0.32:8001", + }, + }, + MaxFails: 12, + FailTimeout: "20s", + MaxConns: 4, + UpstreamZoneSize: "256k", + }, + }, + SplitClients: []SplitClient{ + { + Source: "$request_id", + Variable: "$split_0", + Distributions: []Distribution{ + { + Weight: "50%", + Value: "@loc0", + }, + { + Weight: "50%", + Value: "@loc1", + }, + }, + }, + }, + Maps: []Map{ + { + Source: "$match_0_0", + Variable: "$match", + Parameters: []Parameter{ + { + Value: "~^1", + Result: "@match_loc_0", + }, + { + Value: "default", + Result: "@match_loc_default", + }, + }, + }, + { + Source: "$http_x_version", + Variable: "$match_0_0", + Parameters: []Parameter{ + { + Value: "v2", + Result: "1", + }, + { + Value: "default", + Result: "0", + }, + }, + }, + }, + HTTPSnippets: []string{"# HTTP snippet"}, + Server: Server{ + ServerName: "example.com", + StatusZone: "example.com", + ProxyProtocol: true, + SSL: &SSL{ + HTTP2: true, + Certificate: "cafe-secret.pem", + CertificateKey: "cafe-secret.pem", + }, + TLSRedirect: &TLSRedirect{ + BasedOn: "$scheme", + Code: 301, + }, + CustomListeners: true, + HTTPPort: 8082, + HTTPSPort: 0, + ServerTokens: "off", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Allow: []string{"127.0.0.1"}, + Deny: []string{"127.0.0.1"}, + LimitReqs: []LimitReq{ + { + ZoneName: "pol_rl_test_test_test", + Delay: 10, + Burst: 5, + }, + }, + LimitReqOptions: LimitReqOptions{ + LogLevel: "error", + RejectCode: 503, + }, + JWTAuth: &JWTAuth{ + Realm: "My Api", + Secret: "jwk-secret", + }, + IngressMTLS: &IngressMTLS{ + ClientCert: "ingress-mtls-secret", + VerifyClient: "on", + VerifyDepth: 2, + }, + WAF: &WAF{ + ApPolicy: "/etc/nginx/waf/nac-policies/default-dataguard-alarm", + ApSecurityLogEnable: true, + ApLogConf: []string{"/etc/nginx/waf/nac-logconfs/default-logconf"}, + }, + Snippets: []string{"# server snippet"}, + InternalRedirectLocations: []InternalRedirectLocation{ + { + Path: "/split", + Destination: "@split_0", + }, + { + Path: "/coffee", + Destination: "@match", + }, + }, + HealthChecks: []HealthCheck{ + { + Name: "coffee", + URI: "/", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://coffee-v2", + Mandatory: true, + Persistent: true, + }, + { + Name: "tea", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://tea-v2", + GRPCPass: "grpc://tea-v3", + GRPCStatus: createPointerFromInt(12), + GRPCService: "tea-servicev2", + }, + }, + Locations: []Location{ + { + Path: "/", + Snippets: []string{"# location snippet"}, + Allow: []string{"127.0.0.1"}, + Deny: []string{"127.0.0.1"}, + LimitReqs: []LimitReq{ + { + ZoneName: "loc_pol_rl_test_test_test", + }, + }, + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyBuffering: true, + ProxyBuffers: "8 4k", + ProxyBufferSize: "4k", + ProxyMaxTempFileSize: "1024m", + ProxyPass: "http://test-upstream", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + Internal: true, + ProxyPassRequestHeaders: false, + ProxyPassHeaders: []string{"Host"}, + ProxyPassRewrite: "$request_uri", + ProxyHideHeaders: []string{"Header"}, + ProxyIgnoreHeaders: "Cache", + Rewrites: []string{"$request_uri $request_uri", "$request_uri $request_uri"}, + AddHeaders: []AddHeader{ + { + Header: Header{ + Name: "Header-Name", + Value: "Header Value", + }, + Always: true, + }, + }, + EgressMTLS: &EgressMTLS{ + Certificate: "egress-mtls-secret.pem", + CertificateKey: "egress-mtls-secret.pem", + VerifyServer: true, + VerifyDepth: 1, + Ciphers: "DEFAULT", + Protocols: "TLSv1.3", + TrustedCert: "trusted-cert.pem", + SessionReuse: true, + ServerName: true, + }, + }, + { + Path: "@loc0", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + ProxyInterceptErrors: true, + ErrorPages: []ErrorPage{ + { + Name: "@error_page_1", + Codes: "400 500", + ResponseCode: 200, + }, + { + Name: "@error_page_2", + Codes: "500", + ResponseCode: 0, + }, + }, + }, + { + Path: "@loc1", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "@loc2", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + GRPCPass: "grpc://coffee-v3", + }, + { + Path: "@match_loc_0", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "@match_loc_default", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "/return", + ProxyInterceptErrors: true, + ErrorPages: []ErrorPage{ + { + Name: "@return_0", + Codes: "418", + ResponseCode: 200, + }, + }, + InternalProxyPass: "http://unix:/var/lib/nginx/nginx-418-server.sock", + }, + }, + ErrorPageLocations: []ErrorPageLocation{ + { + Name: "@vs_cafe_cafe_vsr_tea_tea_tea__tea_error_page_0", + DefaultType: "application/json", + Return: &Return{ + Code: 200, + Text: "Hello World", + }, + Headers: nil, + }, + { + Name: "@vs_cafe_cafe_vsr_tea_tea_tea__tea_error_page_1", + DefaultType: "", + Return: &Return{ + Code: 200, + Text: "Hello World", + }, + Headers: []Header{ + { + Name: "Set-Cookie", + Value: "cookie1=test", + }, + { + Name: "Set-Cookie", + Value: "cookie2=test; Secure", + }, + }, + }, + }, + ReturnLocations: []ReturnLocation{ + { + Name: "@return_0", + DefaultType: "text/html", + Return: Return{ + Code: 200, + Text: "Hello!", + }, + }, + }, + }, + } + + virtualServerCfgWithCustomListenerHTTPSOnly = VirtualServerConfig{ + LimitReqZones: []LimitReqZone{ + { + ZoneName: "pol_rl_test_test_test", Rate: "10r/s", ZoneSize: "10m", Key: "$url", + }, + }, + Upstreams: []Upstream{ + { + Name: "test-upstream", + Servers: []UpstreamServer{ + { + Address: "10.0.0.20:8001", + }, + }, + LBMethod: "random", + Keepalive: 32, + MaxFails: 4, + FailTimeout: "10s", + MaxConns: 31, + SlowStart: "10s", + UpstreamZoneSize: "256k", + Queue: &Queue{Size: 10, Timeout: "60s"}, + SessionCookie: &SessionCookie{Enable: true, Name: "test", Path: "/tea", Expires: "25s"}, + NTLM: true, + }, + { + Name: "coffee-v1", + Servers: []UpstreamServer{ + { + Address: "10.0.0.31:8001", + }, + }, + MaxFails: 8, + FailTimeout: "15s", + MaxConns: 2, + UpstreamZoneSize: "256k", + }, + { + Name: "coffee-v2", + Servers: []UpstreamServer{ + { + Address: "10.0.0.32:8001", + }, + }, + MaxFails: 12, + FailTimeout: "20s", + MaxConns: 4, + UpstreamZoneSize: "256k", + }, + }, + SplitClients: []SplitClient{ + { + Source: "$request_id", + Variable: "$split_0", + Distributions: []Distribution{ + { + Weight: "50%", + Value: "@loc0", + }, + { + Weight: "50%", + Value: "@loc1", + }, + }, + }, + }, + Maps: []Map{ + { + Source: "$match_0_0", + Variable: "$match", + Parameters: []Parameter{ + { + Value: "~^1", + Result: "@match_loc_0", + }, + { + Value: "default", + Result: "@match_loc_default", + }, + }, + }, + { + Source: "$http_x_version", + Variable: "$match_0_0", + Parameters: []Parameter{ + { + Value: "v2", + Result: "1", + }, + { + Value: "default", + Result: "0", + }, + }, + }, + }, + HTTPSnippets: []string{"# HTTP snippet"}, + Server: Server{ + ServerName: "example.com", + StatusZone: "example.com", + ProxyProtocol: true, + SSL: &SSL{ + HTTP2: true, + Certificate: "cafe-secret.pem", + CertificateKey: "cafe-secret.pem", + }, + TLSRedirect: &TLSRedirect{ + BasedOn: "$scheme", + Code: 301, + }, + CustomListeners: true, + HTTPPort: 0, + HTTPSPort: 8443, + ServerTokens: "off", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Allow: []string{"127.0.0.1"}, + Deny: []string{"127.0.0.1"}, + LimitReqs: []LimitReq{ + { + ZoneName: "pol_rl_test_test_test", + Delay: 10, + Burst: 5, + }, + }, + LimitReqOptions: LimitReqOptions{ + LogLevel: "error", + RejectCode: 503, + }, + JWTAuth: &JWTAuth{ + Realm: "My Api", + Secret: "jwk-secret", + }, + IngressMTLS: &IngressMTLS{ + ClientCert: "ingress-mtls-secret", + VerifyClient: "on", + VerifyDepth: 2, + }, + WAF: &WAF{ + ApPolicy: "/etc/nginx/waf/nac-policies/default-dataguard-alarm", + ApSecurityLogEnable: true, + ApLogConf: []string{"/etc/nginx/waf/nac-logconfs/default-logconf"}, + }, + Snippets: []string{"# server snippet"}, + InternalRedirectLocations: []InternalRedirectLocation{ + { + Path: "/split", + Destination: "@split_0", + }, + { + Path: "/coffee", + Destination: "@match", + }, + }, + HealthChecks: []HealthCheck{ + { + Name: "coffee", + URI: "/", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://coffee-v2", + Mandatory: true, + Persistent: true, + }, + { + Name: "tea", + Interval: "5s", + Jitter: "0s", + Fails: 1, + Passes: 1, + Port: 50, + ProxyPass: "http://tea-v2", + GRPCPass: "grpc://tea-v3", + GRPCStatus: createPointerFromInt(12), + GRPCService: "tea-servicev2", + }, + }, + Locations: []Location{ + { + Path: "/", + Snippets: []string{"# location snippet"}, + Allow: []string{"127.0.0.1"}, + Deny: []string{"127.0.0.1"}, + LimitReqs: []LimitReq{ + { + ZoneName: "loc_pol_rl_test_test_test", + }, + }, + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyBuffering: true, + ProxyBuffers: "8 4k", + ProxyBufferSize: "4k", + ProxyMaxTempFileSize: "1024m", + ProxyPass: "http://test-upstream", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + Internal: true, + ProxyPassRequestHeaders: false, + ProxyPassHeaders: []string{"Host"}, + ProxyPassRewrite: "$request_uri", + ProxyHideHeaders: []string{"Header"}, + ProxyIgnoreHeaders: "Cache", + Rewrites: []string{"$request_uri $request_uri", "$request_uri $request_uri"}, + AddHeaders: []AddHeader{ + { + Header: Header{ + Name: "Header-Name", + Value: "Header Value", + }, + Always: true, + }, + }, + EgressMTLS: &EgressMTLS{ + Certificate: "egress-mtls-secret.pem", + CertificateKey: "egress-mtls-secret.pem", + VerifyServer: true, + VerifyDepth: 1, + Ciphers: "DEFAULT", + Protocols: "TLSv1.3", + TrustedCert: "trusted-cert.pem", + SessionReuse: true, + ServerName: true, + }, + }, + { + Path: "@loc0", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + ProxyInterceptErrors: true, + ErrorPages: []ErrorPage{ + { + Name: "@error_page_1", + Codes: "400 500", + ResponseCode: 200, + }, + { + Name: "@error_page_2", + Codes: "500", + ResponseCode: 0, + }, + }, + }, + { + Path: "@loc1", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "@loc2", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + GRPCPass: "grpc://coffee-v3", + }, + { + Path: "@match_loc_0", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v2", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "@match_loc_default", + ProxyConnectTimeout: "30s", + ProxyReadTimeout: "31s", + ProxySendTimeout: "32s", + ClientMaxBodySize: "1m", + ProxyPass: "http://coffee-v1", + ProxyNextUpstream: "error timeout", + ProxyNextUpstreamTimeout: "5s", + }, + { + Path: "/return", + ProxyInterceptErrors: true, + ErrorPages: []ErrorPage{ + { + Name: "@return_0", + Codes: "418", + ResponseCode: 200, + }, + }, + InternalProxyPass: "http://unix:/var/lib/nginx/nginx-418-server.sock", + }, + }, + ErrorPageLocations: []ErrorPageLocation{ + { + Name: "@vs_cafe_cafe_vsr_tea_tea_tea__tea_error_page_0", + DefaultType: "application/json", + Return: &Return{ + Code: 200, + Text: "Hello World", + }, + Headers: nil, + }, + { + Name: "@vs_cafe_cafe_vsr_tea_tea_tea__tea_error_page_1", + DefaultType: "", + Return: &Return{ + Code: 200, + Text: "Hello World", + }, + Headers: []Header{ + { + Name: "Set-Cookie", + Value: "cookie1=test", + }, + { + Name: "Set-Cookie", + Value: "cookie2=test; Secure", + }, + }, + }, + }, + ReturnLocations: []ReturnLocation{ + { + Name: "@return_0", + DefaultType: "text/html", + Return: Return{ + Code: 200, + Text: "Hello!", + }, + }, + }, + }, + } + transportServerCfg = TransportServerConfig{ Upstreams: []StreamUpstream{ { diff --git a/internal/configs/virtualserver.go b/internal/configs/virtualserver.go index a01fe24eaa..8d0dd95560 100644 --- a/internal/configs/virtualserver.go +++ b/internal/configs/virtualserver.go @@ -76,6 +76,8 @@ type PodInfo struct { // VirtualServerEx holds a VirtualServer along with the resources that are referenced in this VirtualServer. type VirtualServerEx struct { VirtualServer *conf_v1.VirtualServer + HTTPPort int + HTTPSPort int Endpoints map[string][]string VirtualServerRoutes []*conf_v1.VirtualServerRoute ExternalNameSvcs map[string]bool @@ -310,6 +312,12 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( ) (version2.VirtualServerConfig, Warnings) { vsc.clearWarnings() + useCustomListeners := false + + if vsEx.VirtualServer.Spec.Listener != nil { + useCustomListeners = true + } + sslConfig := vsc.generateSSLConfig(vsEx.VirtualServer, vsEx.VirtualServer.Spec.TLS, vsEx.VirtualServer.Namespace, vsEx.SecretRefs, vsc.cfgParams) tlsRedirectConfig := generateTLSRedirectConfig(vsEx.VirtualServer.Spec.TLS) @@ -686,6 +694,9 @@ func (vsc *virtualServerConfigurator) GenerateVirtualServerConfig( ServerName: vsEx.VirtualServer.Spec.Host, Gunzip: vsEx.VirtualServer.Spec.Gunzip, StatusZone: vsEx.VirtualServer.Spec.Host, + HTTPPort: vsEx.HTTPPort, + HTTPSPort: vsEx.HTTPSPort, + CustomListeners: useCustomListeners, ProxyProtocol: vsc.cfgParams.ProxyProtocol, SSL: sslConfig, ServerTokens: vsc.cfgParams.ServerTokens, diff --git a/internal/configs/virtualserver_test.go b/internal/configs/virtualserver_test.go index 8bafb39849..54832aa875 100644 --- a/internal/configs/virtualserver_test.go +++ b/internal/configs/virtualserver_test.go @@ -1272,6 +1272,9 @@ func TestGenerateVirtualServerConfig(t *testing.T) { Server: version2.Server{ ServerName: "cafe.example.com", StatusZone: "cafe.example.com", + HTTPPort: 0, + HTTPSPort: 0, + CustomListeners: false, VSNamespace: "default", VSName: "cafe", ProxyProtocol: true, @@ -1438,6 +1441,198 @@ func TestGenerateVirtualServerConfig(t *testing.T) { } } +func TestGenerateVirtualServerConfigWithCustomHttpAndHttpsListeners(t *testing.T) { + t.Parallel() + + expected := version2.VirtualServerConfig{ + Upstreams: nil, + HTTPSnippets: []string{}, + LimitReqZones: []version2.LimitReqZone{}, + Server: version2.Server{ + ServerName: virtualServerExWithCustomHTTPAndHTTPSListeners.VirtualServer.Spec.Host, + StatusZone: virtualServerExWithCustomHTTPAndHTTPSListeners.VirtualServer.Spec.Host, + VSNamespace: virtualServerExWithCustomHTTPAndHTTPSListeners.VirtualServer.ObjectMeta.Namespace, + VSName: virtualServerExWithCustomHTTPAndHTTPSListeners.VirtualServer.ObjectMeta.Name, + DisableIPV6: true, + HTTPPort: virtualServerExWithCustomHTTPAndHTTPSListeners.HTTPPort, + HTTPSPort: virtualServerExWithCustomHTTPAndHTTPSListeners.HTTPSPort, + CustomListeners: true, + ProxyProtocol: true, + ServerTokens: "off", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Snippets: []string{"# server snippet"}, + Locations: nil, + }, + } + + vsc := newVirtualServerConfigurator( + &baseCfgParams, + false, + false, + &StaticConfigParams{DisableIPV6: true}, + false, + ) + + result, warnings := vsc.GenerateVirtualServerConfig( + &virtualServerExWithCustomHTTPAndHTTPSListeners, + nil, + nil) + + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("GenerateVirtualServerConfig() mismatch (-want +got):\n%s", diff) + } + + if len(warnings) != 0 { + t.Errorf("GenerateVirtualServerConfig returned warnings: %v", vsc.warnings) + } +} + +func TestGenerateVirtualServerConfigWithCustomHttpListener(t *testing.T) { + t.Parallel() + + expected := version2.VirtualServerConfig{ + Upstreams: nil, + HTTPSnippets: []string{}, + LimitReqZones: []version2.LimitReqZone{}, + Server: version2.Server{ + ServerName: virtualServerExWithCustomHTTPListener.VirtualServer.Spec.Host, + StatusZone: virtualServerExWithCustomHTTPListener.VirtualServer.Spec.Host, + VSNamespace: virtualServerExWithCustomHTTPListener.VirtualServer.ObjectMeta.Namespace, + VSName: virtualServerExWithCustomHTTPListener.VirtualServer.ObjectMeta.Name, + DisableIPV6: true, + HTTPPort: virtualServerExWithCustomHTTPListener.HTTPPort, + HTTPSPort: virtualServerExWithCustomHTTPListener.HTTPSPort, + CustomListeners: true, + ProxyProtocol: true, + ServerTokens: "off", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Snippets: []string{"# server snippet"}, + Locations: nil, + }, + } + + vsc := newVirtualServerConfigurator( + &baseCfgParams, + false, + false, + &StaticConfigParams{DisableIPV6: true}, + false, + ) + + result, warnings := vsc.GenerateVirtualServerConfig( + &virtualServerExWithCustomHTTPListener, + nil, + nil) + + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("GenerateVirtualServerConfig() mismatch (-want +got):\n%s", diff) + } + + if len(warnings) != 0 { + t.Errorf("GenerateVirtualServerConfig returned warnings: %v", vsc.warnings) + } +} + +func TestGenerateVirtualServerConfigWithCustomHttpsListener(t *testing.T) { + t.Parallel() + + expected := version2.VirtualServerConfig{ + Upstreams: nil, + HTTPSnippets: []string{}, + LimitReqZones: []version2.LimitReqZone{}, + Server: version2.Server{ + ServerName: virtualServerExWithCustomHTTPSListener.VirtualServer.Spec.Host, + StatusZone: virtualServerExWithCustomHTTPSListener.VirtualServer.Spec.Host, + VSNamespace: virtualServerExWithCustomHTTPSListener.VirtualServer.ObjectMeta.Namespace, + VSName: virtualServerExWithCustomHTTPSListener.VirtualServer.ObjectMeta.Name, + DisableIPV6: true, + HTTPPort: virtualServerExWithCustomHTTPSListener.HTTPPort, + HTTPSPort: virtualServerExWithCustomHTTPSListener.HTTPSPort, + CustomListeners: true, + ProxyProtocol: true, + ServerTokens: "off", + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Snippets: []string{"# server snippet"}, + Locations: nil, + }, + } + + vsc := newVirtualServerConfigurator( + &baseCfgParams, + false, + false, + &StaticConfigParams{DisableIPV6: true}, + false, + ) + + result, warnings := vsc.GenerateVirtualServerConfig( + &virtualServerExWithCustomHTTPSListener, + nil, + nil) + + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("GenerateVirtualServerConfig() mismatch (-want +got):\n%s", diff) + } + + if len(warnings) != 0 { + t.Errorf("GenerateVirtualServerConfig returned warnings: %v", vsc.warnings) + } +} + +func TestGenerateVirtualServerConfigWithNilListener(t *testing.T) { + t.Parallel() + + expected := version2.VirtualServerConfig{ + Upstreams: nil, + HTTPSnippets: []string{}, + LimitReqZones: []version2.LimitReqZone{}, + Server: version2.Server{ + ServerName: virtualServerExWithNilListener.VirtualServer.Spec.Host, + StatusZone: virtualServerExWithNilListener.VirtualServer.Spec.Host, + VSNamespace: virtualServerExWithNilListener.VirtualServer.ObjectMeta.Namespace, + VSName: virtualServerExWithNilListener.VirtualServer.ObjectMeta.Name, + DisableIPV6: true, + HTTPPort: virtualServerExWithNilListener.HTTPPort, + HTTPSPort: virtualServerExWithNilListener.HTTPSPort, + CustomListeners: false, + ProxyProtocol: true, + ServerTokens: baseCfgParams.ServerTokens, + SetRealIPFrom: []string{"0.0.0.0/0"}, + RealIPHeader: "X-Real-IP", + RealIPRecursive: true, + Snippets: []string{"# server snippet"}, + Locations: nil, + }, + } + + vsc := newVirtualServerConfigurator( + &baseCfgParams, + false, + false, + &StaticConfigParams{DisableIPV6: true}, + false, + ) + + result, warnings := vsc.GenerateVirtualServerConfig( + &virtualServerExWithNilListener, + nil, + nil) + + if diff := cmp.Diff(expected, result); diff != "" { + t.Errorf("GenerateVirtualServerConfig() mismatch (-want +got):\n%s", diff) + } + + if len(warnings) != 0 { + t.Errorf("GenerateVirtualServerConfig returned warnings: %v", vsc.warnings) + } +} + func TestGenerateVirtualServerConfigIPV6Disabled(t *testing.T) { t.Parallel() virtualServerEx := VirtualServerEx{ @@ -11006,4 +11201,67 @@ var ( }, }, } + + virtualServerExWithCustomHTTPAndHTTPSListeners = VirtualServerEx{ + HTTPPort: 8083, + HTTPSPort: 8443, + VirtualServer: &conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + Spec: conf_v1.VirtualServerSpec{ + Host: "cafe.example.com", + Listener: &conf_v1.Listener{ + HTTP: "http-8083", + HTTPS: "https-8443", + }, + }, + }, + } + + virtualServerExWithCustomHTTPListener = VirtualServerEx{ + HTTPPort: 8083, + VirtualServer: &conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + Spec: conf_v1.VirtualServerSpec{ + Host: "cafe.example.com", + Listener: &conf_v1.Listener{ + HTTP: "http-8083", + }, + }, + }, + } + + virtualServerExWithCustomHTTPSListener = VirtualServerEx{ + HTTPSPort: 8443, + VirtualServer: &conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + Spec: conf_v1.VirtualServerSpec{ + Host: "cafe.example.com", + Listener: &conf_v1.Listener{ + HTTPS: "https-8443", + }, + }, + }, + } + + virtualServerExWithNilListener = VirtualServerEx{ + VirtualServer: &conf_v1.VirtualServer{ + ObjectMeta: meta_v1.ObjectMeta{ + Name: "cafe", + Namespace: "default", + }, + Spec: conf_v1.VirtualServerSpec{ + Host: "cafe.example.com", + Listener: nil, + }, + }, + } ) diff --git a/internal/k8s/configuration.go b/internal/k8s/configuration.go index c98b0a0c31..e23eed1f06 100644 --- a/internal/k8s/configuration.go +++ b/internal/k8s/configuration.go @@ -199,6 +199,8 @@ type VirtualServerConfiguration struct { VirtualServer *conf_v1.VirtualServer VirtualServerRoutes []*conf_v1.VirtualServerRoute Warnings []string + HTTPPort int + HTTPSPort int } // NewVirtualServerConfiguration creates a VirtualServerConfiguration. @@ -325,8 +327,9 @@ type TransportServerMetrics struct { // The IC needs to ensure that at any point in time the NGINX config on the filesystem reflects the state // of the objects in the Configuration. type Configuration struct { - hosts map[string]Resource - listeners map[string]*TransportServerConfiguration + hosts map[string]Resource + listeners map[string]*TransportServerConfiguration + listenerMap map[string]conf_v1alpha1.Listener // only valid resources with the matching IngressClass are stored ingresses map[string]*networking.Ingress @@ -595,6 +598,9 @@ func (c *Configuration) AddOrUpdateGlobalConfiguration(gc *conf_v1alpha1.GlobalC c.lock.Lock() defer c.lock.Unlock() + var changes []ResourceChange + var problems []ConfigurationProblem + validationErr := c.globalConfigurationValidator.ValidateGlobalConfiguration(gc) if validationErr != nil { c.globalConfiguration = nil @@ -602,7 +608,15 @@ func (c *Configuration) AddOrUpdateGlobalConfiguration(gc *conf_v1alpha1.GlobalC c.globalConfiguration = gc } - changes, problems := c.rebuildListeners() + c.setGlobalConfigListenerMap() + + listenerChanges, listenerProblems := c.rebuildListeners() + changes = append(changes, listenerChanges...) + problems = append(problems, listenerProblems...) + + hostChanges, hostProblems := c.rebuildHosts() + changes = append(changes, hostChanges...) + problems = append(problems, hostProblems...) return changes, problems, validationErr } @@ -612,8 +626,18 @@ func (c *Configuration) DeleteGlobalConfiguration() ([]ResourceChange, []Configu c.lock.Lock() defer c.lock.Unlock() + var changes []ResourceChange + var problems []ConfigurationProblem + c.globalConfiguration = nil - changes, problems := c.rebuildListeners() + c.setGlobalConfigListenerMap() + listenerChanges, listenerProblems := c.rebuildListeners() + changes = append(changes, listenerChanges...) + problems = append(problems, listenerProblems...) + + hostChanges, hostProblems := c.rebuildHosts() + changes = append(changes, hostChanges...) + problems = append(problems, hostProblems...) return changes, problems } @@ -791,6 +815,23 @@ func (c *Configuration) buildListenersAndTSConfigurations() (newListeners map[st return newListeners, newTSConfigs } +func (c *Configuration) buildListenersForVSConfiguration(vsc *VirtualServerConfiguration) { + vs := vsc.VirtualServer + if vs.Spec.Listener != nil && c.globalConfiguration != nil { + if gcListener, ok := c.listenerMap[vs.Spec.Listener.HTTP]; ok { + if gcListener.Protocol == conf_v1.HTTPProtocol && !gcListener.Ssl { + vsc.HTTPPort = gcListener.Port + } + } + + if gcListener, ok := c.listenerMap[vs.Spec.Listener.HTTPS]; ok { + if gcListener.Protocol == conf_v1.HTTPProtocol && gcListener.Ssl { + vsc.HTTPSPort = gcListener.Port + } + } + } +} + // GetResources returns all configuration resources. func (c *Configuration) GetResources() []Resource { return c.GetResourcesWithFilter(resourceFilter{ @@ -961,12 +1002,12 @@ func (c *Configuration) rebuildHosts() ([]ResourceChange, []ConfigurationProblem changes[i].Resource = r } } - newProblems := make(map[string]ConfigurationProblem) c.addProblemsForResourcesWithoutActiveHost(newResources, newProblems) c.addProblemsForOrphanMinions(newProblems) c.addProblemsForOrphanOrIgnoredVsrs(newProblems) + c.addWarningsForVirtualServersWithMissConfiguredListeners(newResources) newOrUpdatedProblems := detectChangesInProblems(newProblems, c.hostProblems) @@ -1084,6 +1125,61 @@ func (c *Configuration) addProblemsForResourcesWithoutActiveHost(resources map[s } } +func (c *Configuration) addWarningsForVirtualServersWithMissConfiguredListeners(resources map[string]Resource) { + for _, r := range resources { + vsc, ok := r.(*VirtualServerConfiguration) + if !ok { + continue + } + if vsc.VirtualServer.Spec.Listener != nil { + if c.globalConfiguration == nil { + warningMsg := "Listeners defined, but no GlobalConfiguration is deployed" + c.hosts[vsc.VirtualServer.Spec.Host].AddWarning(warningMsg) + continue + } + + if !c.isListenerInCorrectBlock(vsc.VirtualServer.Spec.Listener.HTTP, false) { + warningMsg := fmt.Sprintf("Listener %s can't be use in `listener.http` context as SSL is enabled for that listener.", + vsc.VirtualServer.Spec.Listener.HTTP) + c.hosts[vsc.VirtualServer.Spec.Host].AddWarning(warningMsg) + continue + } + + if !c.isListenerInCorrectBlock(vsc.VirtualServer.Spec.Listener.HTTPS, true) { + warningMsg := fmt.Sprintf("Listener %s can't be use in `listener.https` context as SSL is not enabled for that listener.", + vsc.VirtualServer.Spec.Listener.HTTPS) + c.hosts[vsc.VirtualServer.Spec.Host].AddWarning(warningMsg) + continue + } + + if vsc.VirtualServer.Spec.Listener.HTTP != "" { + if _, exists := c.listenerMap[vsc.VirtualServer.Spec.Listener.HTTP]; !exists { + warningMsg := fmt.Sprintf("Listener %s is not defined in GlobalConfiguration", + vsc.VirtualServer.Spec.Listener.HTTP) + c.hosts[vsc.VirtualServer.Spec.Host].AddWarning(warningMsg) + continue + } + } + + if vsc.VirtualServer.Spec.Listener.HTTPS != "" { + if _, exists := c.listenerMap[vsc.VirtualServer.Spec.Listener.HTTPS]; !exists { + warningMsg := fmt.Sprintf("Listener %s is not defined in GlobalConfiguration", + vsc.VirtualServer.Spec.Listener.HTTPS) + c.hosts[vsc.VirtualServer.Spec.Host].AddWarning(warningMsg) + continue + } + } + } + } +} + +func (c *Configuration) isListenerInCorrectBlock(listenerName string, expectedSsl bool) bool { + if listener, ok := c.listenerMap[listenerName]; listener.Ssl != expectedSsl && ok { + return false + } + return true +} + func (c *Configuration) addProblemsForOrphanMinions(problems map[string]ConfigurationProblem) { for _, key := range getSortedIngressKeys(c.ingresses) { ing := c.ingresses[key] @@ -1346,6 +1442,8 @@ func (c *Configuration) buildHostsAndResources() (newHosts map[string]Resource, } resource := NewVirtualServerConfiguration(vs, vsrs, warnings) + c.buildListenersForVSConfiguration(resource) + newResources[resource.GetKeyWithKind()] = resource holder, exists := newHosts[vs.Spec.Host] @@ -1556,6 +1654,16 @@ func (c *Configuration) GetTransportServerMetrics() *TransportServerMetrics { return &metrics } +func (c *Configuration) setGlobalConfigListenerMap() { + c.listenerMap = make(map[string]conf_v1alpha1.Listener) + + if c.globalConfiguration != nil { + for _, listener := range c.globalConfiguration.Spec.Listeners { + c.listenerMap[listener.Name] = listener + } + } +} + func getSortedIngressKeys(m map[string]*networking.Ingress) []string { var keys []string @@ -1660,9 +1768,19 @@ func detectChangesInHosts(oldHosts map[string]Resource, newHosts map[string]Reso if !exists { continue } - if !oldR.IsEqual(newHosts[h]) { updatedHosts = append(updatedHosts, h) + continue + } + + newVsc, newHostOk := newHosts[h].(*VirtualServerConfiguration) + oldVsc, oldHostOk := oldHosts[h].(*VirtualServerConfiguration) + if !newHostOk || !oldHostOk { + continue + } + + if newVsc.HTTPPort != oldVsc.HTTPPort || newVsc.HTTPSPort != oldVsc.HTTPSPort { + updatedHosts = append(updatedHosts, h) } } diff --git a/internal/k8s/configuration_test.go b/internal/k8s/configuration_test.go index bf18d4c3d9..e7b1b81398 100644 --- a/internal/k8s/configuration_test.go +++ b/internal/k8s/configuration_test.go @@ -1,7 +1,6 @@ package k8s import ( - "fmt" "testing" "time" @@ -1887,8 +1886,8 @@ func TestAddTransportServer(t *testing.T) { Protocol: "TCP", }, } - gc := createTestGlobalConfiguration(listeners) - mustInitGlobalConfiguration(configuration, gc) + + addOrUpdateGlobalConfiguration(t, configuration, listeners, noChanges, noProblems) ts := createTestTransportServer("transportserver", "tcp-7777", "TCP") @@ -2069,8 +2068,7 @@ func TestListenerFlip(t *testing.T) { Protocol: "TCP", }, } - gc := createTestGlobalConfiguration(listeners) - mustInitGlobalConfiguration(configuration, gc) + addOrUpdateGlobalConfiguration(t, configuration, listeners, noChanges, noProblems) ts := createTestTransportServer("transportserver", "tcp-7777", "TCP") @@ -2247,8 +2245,7 @@ func TestAddTransportServerWithIncorrectClass(t *testing.T) { func TestAddTransportServerWithNonExistingListener(t *testing.T) { configuration := createTestConfiguration() - gc := createTestGlobalConfiguration([]conf_v1alpha1.Listener{}) - mustInitGlobalConfiguration(configuration, gc) + addOrUpdateGlobalConfiguration(t, configuration, []conf_v1alpha1.Listener{}, noChanges, noProblems) ts := createTestTransportServer("transportserver", "tcp-7777", "TCP") @@ -2286,7 +2283,40 @@ func TestDeleteNonExistingTransportServer(t *testing.T) { } } -func TestAddGlobalConfiguration(t *testing.T) { +func TestAddOrUpdateGlobalConfiguration(t *testing.T) { + configuration := createTestConfiguration() + + listeners := []conf_v1alpha1.Listener{ + { + Name: "tcp-7777", + Port: 7777, + Protocol: "TCP", + }, + { + Name: "tcp-8888", + Port: 8888, + Protocol: "TCP", + }, + } + gc := createTestGlobalConfiguration(listeners) + + var expectedChanges []ResourceChange + var expectedProblems []ConfigurationProblem + + changes, problems, err := configuration.AddOrUpdateGlobalConfiguration(gc) + if diff := cmp.Diff(expectedChanges, changes); diff != "" { + t.Errorf("AddOrUpdateGlobalConfiguration() returned unexpected result (-want +got):\n%s", diff) + } + if diff := cmp.Diff(expectedProblems, problems); diff != "" { + t.Errorf("AddOrUpdateGlobalConfiguration() returned unexpected result (-want +got):\n%s", diff) + } + if err != nil { + t.Errorf("AddOrUpdateGlobalConfiguration() returned unexpected error: %v", err) + } +} + +//gocyclo:ignore +func TestAddOrUpdateGlobalConfigurationThenAddTransportServer(t *testing.T) { configuration := createTestConfiguration() listeners := []conf_v1alpha1.Listener{ @@ -2571,6 +2601,656 @@ func TestAddGlobalConfiguration(t *testing.T) { } } +func TestAddGlobalConfigurationThenAddVirtualServerWithValidCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithValidCustomListenersFirstThenAddGlobalConfiguration(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 0, + Warnings: []string{"Listeners defined, but no GlobalConfiguration is deployed"}, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithValidCustomListenersAndNoGlobalConfiguration(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 0, + Warnings: []string{"Listeners defined, but no GlobalConfiguration is deployed"}, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithCustomHttpListenerThatDoNotExistInGlobalConfiguration(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-bogus", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 8442, + Warnings: []string{"Listener http-bogus is not defined in GlobalConfiguration"}, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithCustomHttpsListenerThatDoNotExistInGlobalConfiguration(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-bogus") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 0, + Warnings: []string{"Listener https-bogus is not defined in GlobalConfiguration"}, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestDeleteHttpListenerFromExistingGlobalConfigurationWithVirtualServerDeployedWithValidCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 8442, + Warnings: []string{"Listener http-8082 is not defined in GlobalConfiguration"}, + }, + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPSListener, expectedChanges, noProblems) +} + +func TestDeleteHttpsListenerFromExistingGlobalConfigurationWithVirtualServerDeployedWithValidCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 0, + Warnings: []string{"Listener https-8442 is not defined in GlobalConfiguration"}, + }, + }, + } + addOrUpdateGlobalConfiguration(t, configuration, customHTTPListener, expectedChanges, noProblems) +} + +func TestDeleteGlobalConfigurationWithVirtualServerDeployedWithValidCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 0, + Warnings: []string{"Listeners defined, but no GlobalConfiguration is deployed"}, + }, + }, + } + deleteGlobalConfiguration(t, configuration, expectedChanges, noProblems) +} + +func TestRenameHttpListenerInExistingGlobalConfigurationWithVirtualServerDeployedWithValidCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners("cafe", "cafe.example.com", "http-8082", "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 8442, + Warnings: []string{"Listener http-8082 is not defined in GlobalConfiguration"}, + }, + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, bogusHTTPListener, expectedChanges, noProblems) +} + +func TestRenameHttpsListenerInExistingGlobalConfigurationWithVirtualServerDeployedWithValidCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners("cafe", "cafe.example.com", "http-8082", "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 0, + Warnings: []string{"Listener https-8442 is not defined in GlobalConfiguration"}, + }, + }, + } + addOrUpdateGlobalConfiguration(t, configuration, bogusHTTPSListener, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithCustomHttpListenerInHttpsBlock(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "http-8082") + + expectedWarningMsg := "Listener http-8082 can't be use in `listener.https` context as SSL is not enabled for that listener." + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 0, + Warnings: []string{expectedWarningMsg}, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithCustomHttpsListenerInHttpBlock(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "https-8442", + "https-8442") + + expectedWarningMsg := "Listener https-8442 can't be use in `listener.http` context as SSL is enabled for that listener." + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 8442, + Warnings: []string{expectedWarningMsg}, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithNoHttpsListener(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 0, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithNoHttpListener(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithNoHttpOrHttpsListener(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "", + "") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 0, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithValidCustomListenersAndChangeValueOfSslToFalseInGlobalConfiguration(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedWarningMsg := "Listener https-8442 can't be use in `listener.https` context as SSL is not enabled for that listener." + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 0, + Warnings: []string{expectedWarningMsg}, + }, + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPSListenerSSLFalse, expectedChanges, noProblems) +} + +func TestAddVirtualServerWithValidCustomListenersAndChangeValueOfSslToTrueInGlobalConfiguration(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + expectedWarningMsg := "Listener http-8082 can't be use in `listener.http` context as SSL is enabled for that listener." + + expectedChanges = []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 8442, + Warnings: []string{expectedWarningMsg}, + }, + }, + } + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPListenerSSLTrue, expectedChanges, noProblems) +} + +func TestAddMultipleVirtualServersWithTheSameCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServerCafe := createTestVirtualServerWithListeners( + "cafe", + "cafe.example.com", + "http-8082", + "https-8442") + + virtualServerFoo := createTestVirtualServerWithListeners( + "foo", + "foo.example.com", + "http-8082", + "https-8442") + + expectedChangesForVsCafe := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServerCafe, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + addOrUpdateVirtualServer(t, configuration, virtualServerCafe, expectedChangesForVsCafe, noProblems) + + expectedChangesForVsFoo := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServerFoo, + HTTPPort: 8082, + HTTPSPort: 8442, + }, + }, + } + addOrUpdateVirtualServer(t, configuration, virtualServerFoo, expectedChangesForVsFoo, noProblems) +} + +func TestUpdateGlobalConfigurationWithVirtualServerDeployedWithNoCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServer( + "cafe", + "cafe.example.com") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 0, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + // Update GlobalConfiguration by removing a listener + // Since our VirtualServer does not have any listener + // we do not want to see any VirtualServerConfiguration events + addOrUpdateGlobalConfiguration(t, configuration, customHTTPListener, noChanges, noProblems) +} + +func TestDeleteGlobalConfigurationWithVirtualServerDeployedWithNoCustomListeners(t *testing.T) { + t.Parallel() + configuration := createTestConfiguration() + + addOrUpdateGlobalConfiguration(t, configuration, customHTTPAndHTTPSListeners, noChanges, noProblems) + + virtualServer := createTestVirtualServer( + "cafe", + "cafe.example.com") + + expectedChanges := []ResourceChange{ + { + Op: AddOrUpdate, + Resource: &VirtualServerConfiguration{ + VirtualServer: virtualServer, + HTTPPort: 0, + HTTPSPort: 0, + }, + }, + } + + addOrUpdateVirtualServer(t, configuration, virtualServer, expectedChanges, noProblems) + + // Delete the existing GlobalConfiguration + // Since our VirtualServer does not have any listeners + // we do not want to see any VirtualServerConfiguration events + deleteGlobalConfiguration(t, configuration, noChanges, noProblems) +} + func TestPortCollisions(t *testing.T) { configuration := createTestConfiguration() @@ -2581,8 +3261,7 @@ func TestPortCollisions(t *testing.T) { Protocol: "TCP", }, } - gc := createTestGlobalConfiguration(listeners) - mustInitGlobalConfiguration(configuration, gc) + addOrUpdateGlobalConfiguration(t, configuration, listeners, noChanges, noProblems) var expectedChanges []ResourceChange var expectedProblems []ConfigurationProblem @@ -2793,19 +3472,50 @@ func TestChallengeIngressNoVSR(t *testing.T) { } } -func mustInitGlobalConfiguration(c *Configuration, gc *conf_v1alpha1.GlobalConfiguration) { +func addOrUpdateGlobalConfiguration(t *testing.T, c *Configuration, listeners []conf_v1alpha1.Listener, expectedChanges []ResourceChange, expectedProblems []ConfigurationProblem) { + t.Helper() + gc := createTestGlobalConfiguration(listeners) changes, problems, err := c.AddOrUpdateGlobalConfiguration(gc) - // when adding a valid GlobalConfiguration to a new Configuration, no changes, problems and errors are expected - - if len(changes) > 0 { - panic(fmt.Sprintf("AddOrUpdateGlobalConfiguration() returned %d changes, expected 0", len(changes))) + if !cmp.Equal(expectedChanges, changes) { + t.Fatalf("AddOrUpdateGlobalConfiguration() returned unexpected result (-want +got):\n%s", + cmp.Diff(expectedChanges, changes)) } - if len(problems) > 0 { - panic(fmt.Sprintf("AddOrUpdateGlobalConfiguration() returned %d problems, expected 0", len(problems))) + + if !cmp.Equal(expectedProblems, problems) { + t.Fatalf("AddOrUpdateGlobalConfiguration() returned unexpected result (-want +got):\n%s", + cmp.Diff(expectedProblems, problems)) } if err != nil { - panic(fmt.Sprintf("AddOrUpdateGlobalConfiguration() returned an unexpected error %v", err)) + t.Fatalf("AddOrUpdateGlobalConfiguration() returned an unexpected error %v", err) + } +} + +func deleteGlobalConfiguration(t *testing.T, c *Configuration, expectedChanges []ResourceChange, expectedProblems []ConfigurationProblem) { + t.Helper() + changes, problems := c.DeleteGlobalConfiguration() + if !cmp.Equal(expectedChanges, changes) { + t.Fatalf("DeleteGlobalConfiguration() returned unexpected result (-want +got):\n%s", + cmp.Diff(expectedChanges, changes)) + } + + if !cmp.Equal(expectedProblems, problems) { + t.Fatalf("DeleteGlobalConfiguration() returned unexpected result (-want +got):\n%s", + cmp.Diff(expectedProblems, problems)) + } +} + +func addOrUpdateVirtualServer(t *testing.T, c *Configuration, virtualServer *conf_v1.VirtualServer, expectedChanges []ResourceChange, expectedProblems []ConfigurationProblem) { + changes, problems := c.AddOrUpdateVirtualServer(virtualServer) + + if !cmp.Equal(expectedChanges, changes) { + t.Fatalf("AddOrUpdateVirtualServer() returned unexpected result (-want +got):\n%s", + cmp.Diff(expectedChanges, changes)) + } + + if !cmp.Equal(expectedProblems, problems) { + t.Fatalf("AddOrUpdateVirtualServer() returned unexpected result (-want +got):\n%s", + cmp.Diff(noProblems, problems)) } } @@ -2915,6 +3625,24 @@ func createTestVirtualServer(name string, host string) *conf_v1.VirtualServer { } } +func createTestVirtualServerWithListeners(name string, host string, httpListener string, httpsListener string) *conf_v1.VirtualServer { + return &conf_v1.VirtualServer{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: name, + CreationTimestamp: metav1.Now(), + }, + Spec: conf_v1.VirtualServerSpec{ + Listener: &conf_v1.Listener{ + HTTP: httpListener, + HTTPS: httpsListener, + }, + IngressClass: "nginx", + Host: host, + }, + } +} + func createTestVirtualServerWithRoutes(name string, host string, routes []conf_v1.Route) *conf_v1.VirtualServer { vs := createTestVirtualServer(name, host) vs.Spec.Routes = routes @@ -3803,3 +4531,104 @@ func TestCompareConfigurationProblems(t *testing.T) { } } } + +var ( + noChanges []ResourceChange + noProblems []ConfigurationProblem + + // customHTTPAndHTTPSListeners defines a custom HTTP and HTTPS listener on port 8082 and 8442 + customHTTPAndHTTPSListeners = []conf_v1alpha1.Listener{ + { + Name: "http-8082", + Port: 8082, + Protocol: "HTTP", + }, + { + Name: "https-8442", + Port: 8442, + Protocol: "HTTP", + Ssl: true, + }, + } + + // customHTTPSListener defines a customHTTPS listener on port 8442 + customHTTPSListener = []conf_v1alpha1.Listener{ + { + Name: "https-8442", + Port: 8442, + Protocol: "HTTP", + Ssl: true, + }, + } + + // customHTTPListener defines a custom HTTP listener on port 8082 + customHTTPListener = []conf_v1alpha1.Listener{ + { + Name: "http-8082", + Port: 8082, + Protocol: "HTTP", + }, + } + + // customHTTPListenerSSLTrue defines a custom HTTP listener on port 8082 with SSL set to true + customHTTPListenerSSLTrue = []conf_v1alpha1.Listener{ + { + Name: "http-8082", + Port: 8082, + Protocol: "HTTP", + Ssl: true, + }, + { + Name: "https-8442", + Port: 8442, + Protocol: "HTTP", + Ssl: true, + }, + } + + // customHTTPSListenerSSLFalse defines a custom HTTPS listener on port 8442 with SSL set to false + customHTTPSListenerSSLFalse = []conf_v1alpha1.Listener{ + { + Name: "http-8082", + Port: 8082, + Protocol: "HTTP", + Ssl: false, + }, + { + Name: "https-8442", + Port: 8442, + Protocol: "HTTP", + Ssl: false, + }, + } + + // bogusHTTPListener defines a HTTP listener with an invalid name + bogusHTTPListener = []conf_v1alpha1.Listener{ + { + Name: "http-bogus", + Port: 8082, + Protocol: "HTTP", + }, + { + Name: "https-8442", + Port: 8442, + Protocol: "HTTP", + Ssl: true, + }, + } + + // bogusHTTPsListener defines a HTTPs listener with an invalid name + bogusHTTPSListener = []conf_v1alpha1.Listener{ + { + Name: "http-8082", + Port: 8082, + Protocol: "HTTP", + }, + { + Name: "https-bogus", + Port: 8442, + Protocol: "HTTP", + Ssl: true, + }, + } +) diff --git a/internal/k8s/controller.go b/internal/k8s/controller.go index 72d9348c32..1aaa720ae1 100644 --- a/internal/k8s/controller.go +++ b/internal/k8s/controller.go @@ -994,6 +994,7 @@ func (lbc *LoadBalancerController) sync(task task) { case globalConfiguration: lbc.syncGlobalConfiguration(task) lbc.updateTransportServerMetrics() + lbc.updateVirtualServerMetrics() case transportserver: lbc.syncTransportServer(task) lbc.updateTransportServerMetrics() @@ -1615,30 +1616,55 @@ func (lbc *LoadBalancerController) processChanges(changes []ResourceChange) { // Such changes need to be processed at once to prevent any inconsistencies in the generated NGINX config. func (lbc *LoadBalancerController) processChangesFromGlobalConfiguration(changes []ResourceChange) error { var updatedTSExes []*configs.TransportServerEx - var deletedKeys []string + var updatedVSExes []*configs.VirtualServerEx + var deletedTSKeys []string + var deletedVSKeys []string var updatedResources []Resource for _, c := range changes { - tsConfig := c.Resource.(*TransportServerConfiguration) + switch impl := c.Resource.(type) { + case *VirtualServerConfiguration: + if c.Op == AddOrUpdate { + vsEx := lbc.createVirtualServerEx(impl.VirtualServer, impl.VirtualServerRoutes) - if c.Op == AddOrUpdate { - tsEx := lbc.createTransportServerEx(tsConfig.TransportServer, tsConfig.ListenerPort) + updatedVSExes = append(updatedVSExes, vsEx) + updatedResources = append(updatedResources, impl) + } else if c.Op == Delete { + key := getResourceKey(&impl.VirtualServer.ObjectMeta) - updatedTSExes = append(updatedTSExes, tsEx) - updatedResources = append(updatedResources, tsConfig) - } else if c.Op == Delete { - key := getResourceKey(&tsConfig.TransportServer.ObjectMeta) + deletedVSKeys = append(deletedVSKeys, key) + } + case *TransportServerConfiguration: + if c.Op == AddOrUpdate { + tsEx := lbc.createTransportServerEx(impl.TransportServer, impl.ListenerPort) + + updatedTSExes = append(updatedTSExes, tsEx) + updatedResources = append(updatedResources, impl) + } else if c.Op == Delete { + key := getResourceKey(&impl.TransportServer.ObjectMeta) - deletedKeys = append(deletedKeys, key) + deletedTSKeys = append(deletedTSKeys, key) + } } } var updateErr error - updateErrs := lbc.configurator.UpdateTransportServers(updatedTSExes, deletedKeys) - if len(updateErrs) > 0 { - updateErr = fmt.Errorf("errors received from updating TransportServers after GlobalConfiguration change: %v", updateErrs) + if len(updatedTSExes) > 0 || len(deletedTSKeys) > 0 { + tsUpdateErrs := lbc.configurator.UpdateTransportServers(updatedTSExes, deletedTSKeys) + + if len(tsUpdateErrs) > 0 { + updateErr = fmt.Errorf("errors received from updating TransportServers after GlobalConfiguration change: %v", tsUpdateErrs) + } + } + + if len(updatedVSExes) > 0 || len(deletedVSKeys) > 0 { + vsUpdateErrs := lbc.configurator.UpdateVirtualServers(updatedVSExes, deletedVSKeys) + + if len(vsUpdateErrs) > 0 { + updateErr = fmt.Errorf("errors received from updating VirtualSrvers after GlobalConfiguration change: %v", vsUpdateErrs) + } } lbc.updateResourcesStatusAndEvents(updatedResources, configs.Warnings{}, updateErr) @@ -2928,6 +2954,12 @@ func (lbc *LoadBalancerController) createVirtualServerEx(virtualServer *conf_v1. DosProtectedEx: make(map[string]*configs.DosEx), } + resource := lbc.configuration.hosts[virtualServer.Spec.Host] + if vsc, ok := resource.(*VirtualServerConfiguration); ok { + virtualServerEx.HTTPPort = vsc.HTTPPort + virtualServerEx.HTTPSPort = vsc.HTTPSPort + } + if virtualServer.Spec.TLS != nil && virtualServer.Spec.TLS.Secret != "" { secretKey := virtualServer.Namespace + "/" + virtualServer.Spec.TLS.Secret diff --git a/pkg/apis/configuration/v1/types.go b/pkg/apis/configuration/v1/types.go index 36b857e2d7..5c88b915bb 100644 --- a/pkg/apis/configuration/v1/types.go +++ b/pkg/apis/configuration/v1/types.go @@ -11,6 +11,8 @@ const ( StateValid = "Valid" // StateInvalid is used when the resource failed validation or NGINX failed to reload the corresponding config. StateInvalid = "Invalid" + // HTTPProtocol defines a constant for the HTTP protocol in GlobalConfinguration. + HTTPProtocol = "HTTP" ) // +genclient @@ -38,6 +40,7 @@ type VirtualServer struct { type VirtualServerSpec struct { IngressClass string `json:"ingressClassName"` Host string `json:"host"` + Listener *Listener `json:"listener"` TLS *TLS `json:"tls"` Gunzip bool `json:"gunzip"` Policies []PolicyReference `json:"policies"` @@ -51,6 +54,12 @@ type VirtualServerSpec struct { InternalRoute bool `json:"internalRoute"` } +// Listener references a custom http and/or https listener defined in GlobalConfiguration. +type Listener struct { + HTTP string `json:"http"` + HTTPS string `json:"https"` +} + // ExternalDNS defines externaldns sub-resource of a virtual server. type ExternalDNS struct { Enable bool `json:"enable"` diff --git a/pkg/apis/configuration/v1alpha1/types.go b/pkg/apis/configuration/v1alpha1/types.go index 618e7b02b0..bab9796c3b 100644 --- a/pkg/apis/configuration/v1alpha1/types.go +++ b/pkg/apis/configuration/v1alpha1/types.go @@ -34,6 +34,7 @@ type Listener struct { Name string `json:"name"` Port int `json:"port"` Protocol string `json:"protocol"` + Ssl bool `json:"ssl"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/apis/configuration/validation/globalconfiguration.go b/pkg/apis/configuration/validation/globalconfiguration.go index ccee08d1f8..b6333f671f 100644 --- a/pkg/apis/configuration/validation/globalconfiguration.go +++ b/pkg/apis/configuration/validation/globalconfiguration.go @@ -2,6 +2,8 @@ package validation import ( "fmt" + "sort" + "strings" "github.com/nginxinc/kubernetes-ingress/pkg/apis/configuration/v1alpha1" "k8s.io/apimachinery/pkg/util/sets" @@ -9,6 +11,12 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" ) +var allowedProtocols = map[string]bool{ + "TCP": true, + "UDP": true, + "HTTP": true, +} + // GlobalConfigurationValidator validates a GlobalConfiguration resource. type GlobalConfigurationValidator struct { forbiddenListenerPorts map[int]bool @@ -37,6 +45,8 @@ func (gcv *GlobalConfigurationValidator) validateListeners(listeners []v1alpha1. listenerNames := sets.Set[string]{} portProtocolCombinations := sets.Set[string]{} + portProtocolMap := make(map[int]string) + for i, l := range listeners { idxPath := fieldPath.Index(i) portProtocolKey := generatePortProtocolKey(l.Port, l.Protocol) @@ -49,9 +59,28 @@ func (gcv *GlobalConfigurationValidator) validateListeners(listeners []v1alpha1. } else if portProtocolCombinations.Has(portProtocolKey) { msg := fmt.Sprintf("Duplicated port/protocol combination %s", portProtocolKey) allErrs = append(allErrs, field.Duplicate(fieldPath, msg)) + } else if protocol, ok := portProtocolMap[l.Port]; ok { + var msg string + switch protocol { + case "HTTP": + if l.Protocol == "TCP" || l.Protocol == "UDP" { + msg = fmt.Sprintf( + "Listener %s with protocol %s can't use port %d. Port is taken by an HTTP listener", + l.Name, l.Protocol, l.Port) + allErrs = append(allErrs, field.Forbidden(fieldPath, msg)) + } + case "TCP", "UDP": + if l.Protocol == "HTTP" { + msg = fmt.Sprintf( + "Listener %s with protocol %s can't use port %d. Port is taken by TCP or UDP listener", + l.Name, l.Protocol, l.Port) + allErrs = append(allErrs, field.Forbidden(fieldPath, msg)) + } + } } else { listenerNames.Insert(l.Name) portProtocolCombinations.Insert(portProtocolKey) + portProtocolMap[l.Port] = l.Protocol } } @@ -89,3 +118,26 @@ func (gcv *GlobalConfigurationValidator) validateListenerPort(port int, fieldPat } return allErrs } + +func validateListenerProtocol(protocol string, fieldPath *field.Path) field.ErrorList { + switch { + case allowedProtocols[protocol]: + return nil + default: + msg := fmt.Sprintf("must specify a valid protocol. Accepted values: %s", + strings.Join(getProtocolsFromMap(allowedProtocols), ",")) + return field.ErrorList{field.Invalid(fieldPath, protocol, msg)} + } +} + +func getProtocolsFromMap(p map[string]bool) []string { + var keys []string + + for k := range p { + keys = append(keys, k) + } + + sort.Strings(keys) + + return keys +} diff --git a/pkg/apis/configuration/validation/globalconfiguration_test.go b/pkg/apis/configuration/validation/globalconfiguration_test.go index 6d9309c79d..3a3283eefd 100644 --- a/pkg/apis/configuration/validation/globalconfiguration_test.go +++ b/pkg/apis/configuration/validation/globalconfiguration_test.go @@ -209,3 +209,211 @@ func TestGeneratePortProtocolKey(t *testing.T) { t.Errorf("generatePortProtocolKey(%d, %q) returned %q but expected %q", port, protocol, result, expected) } } + +func TestValidateListenerProtocol_FailsOnInvalidInput(t *testing.T) { + t.Parallel() + invalidProtocols := []string{ + "", + "udp", + "UDP ", + } + + for _, p := range invalidProtocols { + allErrs := validateListenerProtocol(p, field.NewPath("protocol")) + if len(allErrs) == 0 { + t.Errorf("validateListenerProtocol(%q) returned no errors for invalid input", p) + } + } +} + +func TestValidateListenerProtocol_PassesOnValidInput(t *testing.T) { + t.Parallel() + validProtocols := []string{ + "TCP", + "HTTP", + "UDP", + } + + for _, p := range validProtocols { + allErrs := validateListenerProtocol(p, field.NewPath("protocol")) + if len(allErrs) != 0 { + t.Errorf("validateListenerProtocol(%q) returned errors for valid input", p) + } + } +} + +func TestValidateListenerProtocol_PassesOnHttpListenerUsingDiffPortToTCPAndUDPListenerWithTCPAndUDPDefinedFirst(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "tcp-listener", + Port: 53, + Protocol: "TCP", + }, + { + Name: "udp-listener", + Port: 53, + Protocol: "UDP", + }, + { + Name: "http-listener", + Port: 63, + Protocol: "HTTP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) > 0 { + t.Errorf("validateListeners() returned errors %v for valid input", allErrs) + } +} + +func TestValidateListenerProtocol_PassesOnHttpListenerUsingDiffPortToTCPAndUDPListenerWithHTTPDefinedFirst(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "http-listener", + Port: 63, + Protocol: "HTTP", + }, + { + Name: "tcp-listener", + Port: 53, + Protocol: "TCP", + }, + { + Name: "udp-listener", + Port: 53, + Protocol: "UDP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) > 0 { + t.Errorf("validateListeners() returned errors %v for valid input", allErrs) + } +} + +func TestValidateListenerProtocol_FailsOnHttpListenerUsingSamePortAsTCPListener(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "tcp-listener", + Port: 53, + Protocol: "TCP", + }, + { + Name: "http-listener", + Port: 53, + Protocol: "HTTP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) == 0 { + t.Errorf("validateListeners() returned no errors %v for invalid input", allErrs) + } +} + +func TestValidateListenerProtocol_FailsOnHttpListenerUsingSamePortAsUDPListener(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "udp-listener", + Port: 53, + Protocol: "UDP", + }, + { + Name: "http-listener", + Port: 53, + Protocol: "HTTP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) == 0 { + t.Errorf("validateListeners() returned no errors %v for invalid input", allErrs) + } +} + +func TestValidateListenerProtocol_FailsOnHttpListenerUsingSamePortAsTCPAndUDPListener(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "tcp-listener", + Port: 53, + Protocol: "TCP", + }, + { + Name: "udp-listener", + Port: 53, + Protocol: "UDP", + }, + { + Name: "http-listener", + Port: 53, + Protocol: "HTTP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) == 0 { + t.Errorf("validateListeners() returned no errors %v for invalid input", allErrs) + } +} + +func TestValidateListenerProtocol_FailsOnTCPListenerUsingSamePortAsHTTPListener(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "http-listener", + Port: 53, + Protocol: "HTTP", + }, + { + Name: "tcp-listener", + Port: 53, + Protocol: "TCP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) == 0 { + t.Errorf("validateListeners() returned no errors %v for invalid input", allErrs) + } +} + +func TestValidateListenerProtocol_FailsOnUDPListenerUsingSamePortAsHTTPListener(t *testing.T) { + t.Parallel() + listeners := []v1alpha1.Listener{ + { + Name: "http-listener", + Port: 53, + Protocol: "HTTP", + }, + { + Name: "udp-listener", + Port: 53, + Protocol: "UDP", + }, + } + + gcv := createGlobalConfigurationValidator() + + allErrs := gcv.validateListeners(listeners, field.NewPath("listeners")) + if len(allErrs) == 0 { + t.Errorf("validateListeners() returned no errors %v for invalid input", allErrs) + } +} diff --git a/pkg/apis/configuration/validation/transportserver.go b/pkg/apis/configuration/validation/transportserver.go index a0d76d4d84..7b975a6934 100644 --- a/pkg/apis/configuration/validation/transportserver.go +++ b/pkg/apis/configuration/validation/transportserver.go @@ -129,15 +129,6 @@ func validateListenerName(name string, fieldPath *field.Path) field.ErrorList { return validateDNS1035Label(name, fieldPath) } -func validateListenerProtocol(protocol string, fieldPath *field.Path) field.ErrorList { - switch protocol { - case "TCP", "UDP": - return nil - default: - return field.ErrorList{field.Invalid(fieldPath, protocol, "must specify protocol. Accepted values: TCP, UDP.")} - } -} - func validateTransportServerUpstreams(upstreams []v1alpha1.Upstream, fieldPath *field.Path, isPlus bool) (allErrs field.ErrorList, upstreamNames sets.Set[string]) { allErrs = field.ErrorList{} upstreamNames = sets.Set[string]{} diff --git a/pkg/apis/configuration/validation/transportserver_test.go b/pkg/apis/configuration/validation/transportserver_test.go index 7b05fc70f8..7e27022b4d 100644 --- a/pkg/apis/configuration/validation/transportserver_test.go +++ b/pkg/apis/configuration/validation/transportserver_test.go @@ -538,23 +538,6 @@ func TestValidateListenerProtocol(t *testing.T) { } } -func TestValidateListenerProtocol_FailsOnInvalidInput(t *testing.T) { - t.Parallel() - invalidProtocols := []string{ - "", - "HTTP", - "udp", - "UDP ", - } - - for _, p := range invalidProtocols { - allErrs := validateListenerProtocol(p, field.NewPath("protocol")) - if len(allErrs) == 0 { - t.Errorf("validateListenerProtocol(%q) returned no errors for invalid input", p) - } - } -} - func TestValidateTSUpstreamHealthChecks(t *testing.T) { t.Parallel() tests := []struct {