Automatically use LB addresses for mesh gateways

Add support for using the external address of Kubernetes load balancer for the mesh gateway wan address. This change uses the new consul-k8s service-address command to get the address of the load balancer. It also removes meshGateway.wanAddress.{useNodeIP, useNodeName, host} config values in favour of meshGateway.wanAddress.{source, static}. The new source value allows selecting NodeIP, NodeName, Service or Static. This is more extensible than the previous boolean values. We also change the default containerPort to 8443 from 443 because that port can't be bound to on GKE. These are backwards incompatible changes.
hashicorp · Mar 26, 2020 · a69bee4 · a69bee4
1 parent 8dd14ff
commit a69bee4
Show file tree

Hide file tree

Showing 6 changed files with 369 additions and 163 deletions.
diff --git a/templates/mesh-gateway-clusterrole.yaml b/templates/mesh-gateway-clusterrole.yaml
@@ -9,7 +9,7 @@ metadata:
     heritage: {{ .Release.Service }}
     release: {{ .Release.Name }}
     component: mesh-gateway
-{{- if or .Values.global.bootstrapACLs .Values.global.enablePodSecurityPolicies }}
+{{- if or .Values.global.bootstrapACLs .Values.global.enablePodSecurityPolicies (eq .Values.meshGateway.wanAddress.source "Service") }}
 rules:
 {{- if .Values.global.enablePodSecurityPolicies }}
   - apiGroups: ["policy"]
@@ -28,6 +28,15 @@ rules:
     verbs:
       - get
 {{- end }}
+{{- if eq .Values.meshGateway.wanAddress.source "Service" }}
+  - apiGroups: [""]
+    resources:
+      - services
+    resourceNames:
+      - {{ template "consul.fullname" . }}-mesh-gateway
+    verbs:
+      - get
+  {{- end }}
 {{- else }}
 rules: []
 {{- end }}

diff --git a/templates/mesh-gateway-deployment.yaml b/templates/mesh-gateway-deployment.yaml
@@ -116,12 +116,35 @@ spec:
                   -token-sink-file=/consul/service/acl-token
                 {{ end }}
 
-                {{- if .Values.meshGateway.wanAddress.host }}
-                WAN_ADDR="{{ .Values.meshGateway.wanAddress.host }}"
-                {{- else if .Values.meshGateway.wanAddress.useNodeName }}
-                WAN_ADDR="${NODE_NAME}"
-                {{- else if .Values.meshGateway.wanAddress.useNodeIP }}
+                {{- $source := .Values.meshGateway.wanAddress.source }}
+                {{- $serviceType := .Values.meshGateway.service.type }}
+                {{- if and (eq $source "Service") (not .Values.meshGateway.service.enabled) }}{{ fail "if meshGateway.wanAddress.source=Service then meshGateway.service.enabled must be set to true" }}{{ end }}
+                {{- if or (eq $source "NodeIP") (and (eq $source "Service") (eq $serviceType "NodePort")) }}
                 WAN_ADDR="${HOST_IP}"
+                {{- else if eq $source "NodeName" }}
+                WAN_ADDR="${NODE_NAME}"
+                {{- else if and (eq $source "Service") (or (eq $serviceType "ClusterIP") (eq $serviceType "LoadBalancer")) }}
+                consul-k8s service-address \
+                  -k8s-namespace={{ .Release.Namespace }} \
+                  -name={{ template "consul.fullname" . }}-mesh-gateway \
+                  -output-file=address.txt
+                WAN_ADDR="$(cat address.txt)"
+                {{- else if eq $source "Static" }}
+                {{- if eq .Values.meshGateway.wanAddress.static "" }}{{ fail "if meshGateway.wanAddress.source=Static then meshGateway.wanAddress.static cannot be empty" }}{{ end }}
+                WAN_ADDR="{{ .Values.meshGateway.wanAddress.static }}"
+                {{- else }}
+                {{- fail "currently set meshGateway values for wanAddress.source and service.type are not supported" }}
+                {{- end }}
+
+                {{- if eq $source "Service" }}
+                {{- if eq $serviceType "NodePort" }}
+                {{- if not .Values.meshGateway.service.nodePort }}{{ fail "if meshGateway.wanAddress.source=Service and meshGateway.service.type=NodePort, meshGateway.service.nodePort must be set" }}{{ end }}
+                WAN_PORT="{{ .Values.meshGateway.service.nodePort }}"
+                {{- else }}
+                WAN_PORT="{{ .Values.meshGateway.service.port }}"
+                {{- end }}
+                {{- else }}
+                WAN_PORT="{{ .Values.meshGateway.wanAddress.port }}"
                 {{- end }}
 
                 cat > /consul/service/service.hcl << EOF
@@ -140,17 +163,9 @@ spec:
                       address = "${POD_IP}"
                       port = {{ .Values.meshGateway.containerPort }}
                     }
-                    lan_ipv4 {
-                      address = "${POD_IP}"
-                      port = {{ .Values.meshGateway.containerPort }}
-                    }
                     wan {
                       address = "${WAN_ADDR}"
-                      port = {{ .Values.meshGateway.wanAddress.port }}
-                    }
-                    wan_ipv4 {
-                      address = "${WAN_ADDR}"
-                      port = {{ .Values.meshGateway.wanAddress.port }}
+                      port = ${WAN_PORT}
                     }
                   }
                   checks = [

diff --git a/test/unit/mesh-gateway-clusterrole.bats b/test/unit/mesh-gateway-clusterrole.bats
@@ -49,19 +49,34 @@ load _helpers
   [ "${actual}" = "secrets" ]
 }
 
-@test "meshGateway/ClusterRole: rules is empty if no ACLs or PSPs" {
+@test "meshGateway/ClusterRole: rules for meshGateway.wanAddress.source=Service" {
   cd `chart_dir`
   local actual=$(helm template \
       -x templates/mesh-gateway-clusterrole.yaml  \
       --set 'meshGateway.enabled=true' \
       --set 'connectInject.enabled=true' \
+      --set 'meshGateway.service.enabled=true' \
+      --set 'meshGateway.service.type=LoadBalancer' \
+      --set 'meshGateway.wanAddress.source=Service' \
+      . | tee /dev/stderr |
+      yq -r '.rules[0].resources[0]' | tee /dev/stderr)
+  [ "${actual}" = "services" ]
+}
+
+@test "meshGateway/ClusterRole: rules is empty if no ACLs, PSPs and meshGateway.source != Service" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -x templates/mesh-gateway-clusterrole.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'meshGateway.wanAddress.source=NodeIP' \
+      --set 'connectInject.enabled=true' \
       --set 'client.grpc=true' \
       . | tee /dev/stderr |
       yq -r '.rules' | tee /dev/stderr)
   [ "${actual}" = "[]" ]
 }
 
-@test "meshGateway/ClusterRole: rules for both ACLs and PSPs" {
+@test "meshGateway/ClusterRole: rules for ACLs, PSPs and mesh gateways" {
   cd `chart_dir`
   local actual=$(helm template \
       -x templates/mesh-gateway-clusterrole.yaml  \
@@ -70,7 +85,10 @@ load _helpers
       --set 'client.grpc=true' \
       --set 'global.bootstrapACLs=true' \
       --set 'global.enablePodSecurityPolicies=true' \
+      --set 'meshGateway.service.enabled=true' \
+      --set 'meshGateway.service.type=LoadBalancer' \
+      --set 'meshGateway.wanAddress.source=Service' \
       . | tee /dev/stderr |
       yq -r '.rules | length' | tee /dev/stderr)
-  [ "${actual}" = "2" ]
+  [ "${actual}" = "3" ]
 }