diff --git a/templates/mesh-gateway-deployment.yaml b/templates/mesh-gateway-deployment.yaml index ea7d8d70e..2f8527a8f 100644 --- a/templates/mesh-gateway-deployment.yaml +++ b/templates/mesh-gateway-deployment.yaml @@ -1,6 +1,7 @@ {{- if .Values.meshGateway.enabled }} {{- if not .Values.connectInject.enabled }}{{ fail "connectInject.enabled must be true" }}{{ end -}} {{- if not .Values.client.grpc }}{{ fail "client.grpc must be true" }}{{ end -}} +{{- if and .Values.global.bootstrapACLs (ne .Values.meshGateway.consulServiceName "") (ne .Values.meshGateway.consulServiceName "mesh-gateway") }}{{ fail "if global.bootstrapACLs is true, meshGateway.consulServiceName cannot be set" }}{{ end -}} {{- /* The below test checks if clients are disabled (and if so, fails). We use the conditional from other client files and prepend 'not' */ -}} {{- if not (or (and (ne (.Values.client.enabled | toString) "-") .Values.client.enabled) (and (eq (.Values.client.enabled | toString) "-") .Values.global.enabled)) }}{{ fail "clients must be enabled" }}{{ end -}} apiVersion: apps/v1 @@ -48,6 +49,8 @@ spec: volumes: - name: consul-bin emptyDir: {} + - name: consul-service + emptyDir: {} {{- if .Values.global.tls.enabled }} - name: consul-ca-cert secret: @@ -79,21 +82,92 @@ spec: volumeMounts: - name: consul-bin mountPath: /consul-bin - {{- if .Values.global.bootstrapACLs }} - # Wait for secret containing acl token to be ready. - # Doesn't do anything with it but when the main container starts we - # know that it's been created. - - name: mesh-gateway-acl-init + # service-init registers the mesh gateway service. + - name: service-init image: {{ .Values.global.imageK8S }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if .Values.global.tls.enabled }} + - name: CONSUL_HTTP_ADDR + value: https://$(HOST_IP):8501 + - name: CONSUL_CACERT + value: /consul/tls/ca/tls.crt + {{- else }} + - name: CONSUL_HTTP_ADDR + value: http://$(HOST_IP):8500 + {{- end }} command: - "/bin/sh" - "-ec" - | - consul-k8s acl-init \ - -secret-name="{{ template "consul.fullname" . }}-mesh-gateway-acl-token" \ - -k8s-namespace={{ .Release.Namespace }} \ - -init-type="sync" - {{- end }} + {{- if .Values.global.bootstrapACLs }} + consul-k8s acl-init \ + -secret-name="{{ template "consul.fullname" . }}-mesh-gateway-acl-token" \ + -k8s-namespace={{ .Release.Namespace }} \ + -init-type="sync" \ + -token-sink-file=/consul/service/acl-token + {{ end }} + + {{- if .Values.meshGateway.wanAddress.host }} + WAN_ADDR="{{ .Values.meshGateway.wanAddress.host }}" + {{- else if .Values.meshGateway.wanAddress.useNodeName }} + WAN_ADDR="${NODE_NAME}" + {{- else if .Values.meshGateway.wanAddress.useNodeIP }} + WAN_ADDR="${HOST_IP}" + {{- end }} + + cat > /consul/service/service.hcl << EOF + service { + kind = "mesh-gateway" + name = "{{ default "mesh-gateway" .Values.meshGateway.consulServiceName }}" + {{- if .Values.global.federation.enabled }} + meta { + consul-wan-federation = "1" + } + {{- end }} + port = {{ .Values.meshGateway.containerPort }} + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = {{ .Values.meshGateway.containerPort }} + } + wan { + address = "${WAN_ADDR}" + port = {{ .Values.meshGateway.wanAddress.port }} + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:{{ .Values.meshGateway.containerPort }}" + deregister_critical_service_after = "6h" + } + ] + } + EOF + + consul services register \ + {{- if .Values.global.bootstrapACLs }} + -token-file=/consul/service/acl-token \ + {{- end }} + /consul/service/service.hcl + volumeMounts: + - name: consul-service + mountPath: /consul/service + {{- if .Values.global.tls.enabled }} + - name: consul-ca-cert + mountPath: /consul/tls/ca + readOnly: true + {{- end }} containers: - name: mesh-gateway image: {{ .Values.meshGateway.imageEnvoy | quote }} @@ -145,29 +219,10 @@ spec: value: $(HOST_IP):8502 {{- end }} command: - # /bin/sh -c is needed so we can use the pod-specific environment - # variables. - - "/bin/sh" - - "-ec" - - | - exec /consul-bin/consul connect envoy \ - -mesh-gateway \ - -register \ - -address="${POD_IP}:{{ .Values.meshGateway.containerPort }}" \ - {{- if .Values.meshGateway.wanAddress.host }} - -wan-address="{{ .Values.meshGateway.wanAddress.host }}:{{ .Values.meshGateway.wanAddress.port }}" \ - {{- else if .Values.meshGateway.wanAddress.useNodeName }} - -wan-address="${NODE_NAME}:{{ .Values.meshGateway.wanAddress.port }}" \ - {{- else if .Values.meshGateway.wanAddress.useNodeIP }} - -wan-address="${HOST_IP}:{{ .Values.meshGateway.wanAddress.port }}" \ - {{- end }} - {{- if and .Values.meshGateway.consulServiceName }} - {{- if and .Values.global.bootstrapACLs (ne .Values.meshGateway.consulServiceName "mesh-gateway") }}{{ fail "if global.bootstrapACLs is true, meshGateway.consulServiceName cannot be set" }}{{ end }} - -service={{ .Values.meshGateway.consulServiceName | quote }} \ - {{- end }} - {{- if .Values.global.federation.enabled }} - -expose-servers \ - {{- end }} + - /consul-bin/consul + - connect + - envoy + - -mesh-gateway {{- if .Values.meshGateway.enableHealthChecks }} livenessProbe: tcpSocket: @@ -197,6 +252,44 @@ spec: exec: command: ["/bin/sh", "-ec", "/consul-bin/consul services deregister -id=\"{{ default "mesh-gateway" .Values.meshGateway.consulServiceName }}\""] + # lifecycle-sidecar ensures the mesh gateway is always registered with + # the local Consul agent, even if it loses the initial registration. + - name: lifecycle-sidecar + image: {{ .Values.global.imageK8S }} + volumeMounts: + - name: consul-service + mountPath: /consul/service + {{- if .Values.global.tls.enabled }} + - name: consul-ca-cert + mountPath: /consul/tls/ca + readOnly: true + {{- end }} + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if .Values.global.tls.enabled }} + - name: CONSUL_HTTP_ADDR + value: https://$(HOST_IP):8501 + - name: CONSUL_CACERT + value: /consul/tls/ca/tls.crt + {{- else }} + - name: CONSUL_HTTP_ADDR + value: http://$(HOST_IP):8500 + {{- end }} + command: + - consul-k8s + - lifecycle-sidecar + - -service-config=/consul/service/service.hcl + - -consul-binary=/bin/consul + {{- if .Values.global.bootstrapACLs }} + - -token-file=/consul/service/acl-token + {{- end }} {{- if .Values.meshGateway.priorityClassName }} priorityClassName: {{ .Values.meshGateway.priorityClassName | quote }} {{- end }} diff --git a/test/unit/mesh-gateway-deployment.bats b/test/unit/mesh-gateway-deployment.bats index f92ec1a60..8427c8f74 100755 --- a/test/unit/mesh-gateway-deployment.bats +++ b/test/unit/mesh-gateway-deployment.bats @@ -243,25 +243,6 @@ key2: value2' \ [ "${actual}" = "ClusterFirst" ] } -#-------------------------------------------------------------------- -# BootstrapACLs - -@test "meshGateway/Deployment: global.BootstrapACLs enabled creates init container and secret" { - cd `chart_dir` - local actual=$(helm template \ - -x templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'client.grpc=true' \ - --set 'global.bootstrapACLs=true' \ - . | tee /dev/stderr ) - local init_container=$(echo "${actual}" | yq -r '.spec.template.spec.initContainers[1].name' | tee /dev/stderr) - [ "${init_container}" = "mesh-gateway-acl-init" ] - - local secret=$(echo "${actual}" | yq -r '.spec.template.spec.containers[0].env[2].name' | tee /dev/stderr) - [ "${secret}" = "CONSUL_HTTP_TOKEN" ] -} - #-------------------------------------------------------------------- # envoyImage @@ -335,7 +316,6 @@ key2: value2' \ . | tee /dev/stderr \ | yq '.spec.template.spec.containers[0]' | tee /dev/stderr) - [ $(echo "$actual" | yq -r '.command | join(" ") | contains("-address=\"${POD_IP}:443\"")') = "true" ] [ $(echo "$actual" | yq -r '.ports[0].containerPort') = "443" ] [ $(echo "$actual" | yq -r '.livenessProbe.tcpSocket.port') = "443" ] [ $(echo "$actual" | yq -r '.readinessProbe.tcpSocket.port') = "443" ] @@ -352,85 +332,11 @@ key2: value2' \ . | tee /dev/stderr \ | yq '.spec.template.spec.containers[0]' | tee /dev/stderr) - [ $(echo "$actual" | yq -r '.command | join(" ") | contains("-address=\"${POD_IP}:8443\"")') = "true" ] [ $(echo "$actual" | yq -r '.ports[0].containerPort') = "8443" ] [ $(echo "$actual" | yq -r '.livenessProbe.tcpSocket.port') = "8443" ] [ $(echo "$actual" | yq -r '.readinessProbe.tcpSocket.port') = "8443" ] } -#-------------------------------------------------------------------- -# wanAddress - -@test "meshGateway/Deployment: wanAddress.port defaults to 443" { - cd `chart_dir` - local actual=$(helm template \ - -x templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'client.grpc=true' \ - --set 'meshGateway.wanAddress.useNodeIP=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].command | join(" ") | contains("-wan-address=\"${HOST_IP}:443\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "meshGateway/Deployment: wanAddress uses NodeIP by default" { - cd `chart_dir` - local actual=$(helm template \ - -x templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'client.grpc=true' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].command | join(" ") | contains("-wan-address=\"${HOST_IP}:443\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "meshGateway/Deployment: wanAddress.useNodeIP" { - cd `chart_dir` - local actual=$(helm template \ - -x templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'client.grpc=true' \ - --set 'meshGateway.wanAddress.useNodeIP=true' \ - --set 'meshGateway.wanAddress.port=4444' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].command | join(" ") | contains("-wan-address=\"${HOST_IP}:4444\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "meshGateway/Deployment: wanAddress.useNodeName" { - cd `chart_dir` - local actual=$(helm template \ - -x templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'client.grpc=true' \ - --set 'meshGateway.wanAddress.useNodeIP=false' \ - --set 'meshGateway.wanAddress.useNodeName=true' \ - --set 'meshGateway.wanAddress.port=4444' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].command | join(" ") | contains("-wan-address=\"${NODE_NAME}:4444\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - -@test "meshGateway/Deployment: wanAddress.host" { - cd `chart_dir` - local actual=$(helm template \ - -x templates/mesh-gateway-deployment.yaml \ - --set 'meshGateway.enabled=true' \ - --set 'connectInject.enabled=true' \ - --set 'client.grpc=true' \ - --set 'meshGateway.wanAddress.useNodeIP=false' \ - --set 'meshGateway.wanAddress.useNodeName=false' \ - --set 'meshGateway.wanAddress.host=myhost' \ - --set 'meshGateway.wanAddress.port=4444' \ - . | tee /dev/stderr | - yq -r '.spec.template.spec.containers[0].command | join(" ") | contains("-wan-address=\"myhost:4444\"")' | tee /dev/stderr) - [ "${actual}" = "true" ] -} - #-------------------------------------------------------------------- # consulServiceName @@ -460,7 +366,6 @@ key2: value2' \ . | tee /dev/stderr \ | yq '.spec.template.spec.containers[0]' | tee /dev/stderr ) - [[ $(echo "${actual}" | yq -r '.command[2]' ) =~ "-service=\"mesh-gateway\"" ]] [[ $(echo "${actual}" | yq -r '.lifecycle.preStop.exec.command' ) =~ '-id=\"mesh-gateway\"' ]] } @@ -475,7 +380,6 @@ key2: value2' \ . | tee /dev/stderr \ | yq '.spec.template.spec.containers[0]' | tee /dev/stderr ) - [[ $(echo "${actual}" | yq -r '.command[2]' ) =~ "-service=\"overridden\"" ]] [[ $(echo "${actual}" | yq -r '.lifecycle.preStop.exec.command' ) =~ '-id=\"overridden\"' ]] } @@ -655,10 +559,106 @@ key2: value2' \ [ "${actual}" = "key" ] } -#-------------------------------------------------------------------- -# global.federation.enabled +##-------------------------------------------------------------------- +## service-init init container -@test "meshGateway/Deployment: -expose-servers set when federation.enabled=true" { +@test "meshGateway/Deployment: service-init init container" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="${HOST_IP}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container with global.bootstrapACLs=true" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'global.bootstrapACLs=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='consul-k8s acl-init \ + -secret-name="release-name-consul-mesh-gateway-acl-token" \ + -k8s-namespace=default \ + -init-type="sync" \ + -token-sink-file=/consul/service/acl-token + +WAN_ADDR="${HOST_IP}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + -token-file=/consul/service/acl-token \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container with global.federation.enabled=true" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ @@ -666,18 +666,270 @@ key2: value2' \ --set 'connectInject.enabled=true' \ --set 'global.federation.enabled=true' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | join(" ") | contains("-expose-servers")' | tee /dev/stderr) - [ "${actual}" = "true" ] + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="${HOST_IP}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + meta { + consul-wan-federation = "1" + } + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] } -@test "meshGateway/Deployment: -expose-servers not set when federation.enabled=false" { +@test "meshGateway/Deployment: service-init init container containerPort and wanAddress.port can be changed" { cd `chart_dir` local actual=$(helm template \ -x templates/mesh-gateway-deployment.yaml \ --set 'meshGateway.enabled=true' \ --set 'connectInject.enabled=true' \ - --set 'global.federation.enabled=false' \ + --set 'meshGateway.containerPort=8888' \ + --set 'meshGateway.wanAddress.port=9999' \ . | tee /dev/stderr | - yq '.spec.template.spec.containers[0].command | join(" ") | contains("-expose-servers")' | tee /dev/stderr) - [ "${actual}" = "false" ] + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="${HOST_IP}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 8888 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 8888 + } + wan { + address = "${WAN_ADDR}" + port = 9999 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:8888" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.useNodeIP" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.useNodeIP=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="${HOST_IP}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.useNodeName" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.useNodeIP=false' \ + --set 'meshGateway.wanAddress.useNodeName=true' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="${NODE_NAME}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container wanAddress.host" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.wanAddress.useNodeIP=false' \ + --set 'meshGateway.wanAddress.host=example.com' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="example.com" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "mesh-gateway" + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] +} + +@test "meshGateway/Deployment: service-init init container consulServiceName can be changed" { + cd `chart_dir` + local actual=$(helm template \ + -x templates/mesh-gateway-deployment.yaml \ + --set 'meshGateway.enabled=true' \ + --set 'connectInject.enabled=true' \ + --set 'meshGateway.consulServiceName=new-name' \ + . | tee /dev/stderr | + yq -r '.spec.template.spec.initContainers | map(select(.name == "service-init"))[0] | .command[2]' | tee /dev/stderr) + + exp='WAN_ADDR="${HOST_IP}" + +cat > /consul/service/service.hcl << EOF +service { + kind = "mesh-gateway" + name = "new-name" + port = 443 + address = "${POD_IP}" + tagged_addresses { + lan { + address = "${POD_IP}" + port = 443 + } + wan { + address = "${WAN_ADDR}" + port = 443 + } + } + checks = [ + { + name = "Mesh Gateway Listening" + interval = "10s" + tcp = "${POD_IP}:443" + deregister_critical_service_after = "6h" + } + ] +} +EOF + +consul services register \ + /consul/service/service.hcl' + + [ "${actual}" = "${exp}" ] }