refactor resource requests and limits for init containers and lifecycle sidecar

templates/connect-inject-deployment.yaml

@@ -138,6 +138,32 @@ spec:
                 {{- if not (kindIs "invalid" $resources.requests.cpu) }}
                 -default-sidecar-proxy-cpu-request={{ $resources.requests.cpu }} \
                 {{- end }}
+                {{- $resources := .Values.global.initContainer.resources }}
+                {{- if not (kindIs "invalid" $resources.limits.memory) }}
+                -init-container-memory-limit={{ $resources.limits.memory }} \
+                {{- end }}
+                {{- if not (kindIs "invalid" $resources.requests.memory) }}
+                -init-container-memory-request={{ $resources.requests.memory }} \
+                {{- end }}
+                {{- if not (kindIs "invalid" $resources.limits.cpu) }}
+                -init-container-cpu-limit={{ $resources.limits.cpu }} \
+                {{- end }}
+                {{- if not (kindIs "invalid" $resources.requests.cpu) }}
+                -init-container-cpu-request={{ $resources.requests.cpu }} \
+                {{- end }}
+                {{- $resources := .Values.global.lifecycleSidecarContainer.resources }}
+                {{- if not (kindIs "invalid" $resources.limits.memory) }}
+                -lifecycle-sidecar-memory-limit={{ $resources.limits.memory }} \
+                {{- end }}
+                {{- if not (kindIs "invalid" $resources.requests.memory) }}
+                -lifecycle-sidecar-memory-request={{ $resources.requests.memory }} \
+                {{- end }}
+                {{- if not (kindIs "invalid" $resources.limits.cpu) }}
+                -lifecycle-sidecar-cpu-limit={{ $resources.limits.cpu }} \
+                {{- end }}
+                {{- if not (kindIs "invalid" $resources.requests.cpu) }}
+                -lifecycle-sidecar-cpu-request={{ $resources.requests.cpu }} \
+                {{- end }}
           livenessProbe:
             httpGet:
               path: /health/ready

templates/ingress-gateways-deployment.yaml

@@ -110,6 +110,7 @@ spec:
           volumeMounts:
           - name: consul-bin
             mountPath: /consul-bin
+          resources: {{ toYaml $root.Values.global.initContainer.resources | nindent 12 }}
         {{- if (and $root.Values.global.tls.enabled $root.Values.global.tls.enableAutoEncrypt) }}
         {{- include "consul.getAutoEncryptClientCA" $root | nindent 8 }}
         {{- end }}
@@ -260,8 +261,7 @@ spec:
         - name: ingress-gateway
           image: {{ $root.Values.global.imageEnvoy | quote }}
           {{- if (default $defaults.resources .resources) }}
-          resources:
-            {{ toYaml (default $defaults.resources .resources) | nindent 12 }}
+          resources: {{ toYaml (default $defaults.resources .resources) | nindent 12 }}
           {{- end }}
           volumeMounts:
           - name: consul-bin
@@ -373,6 +373,7 @@ spec:
               mountPath: /consul/tls/ca
               readOnly: true
             {{- end }}
+          resources: {{ toYaml $root.Values.global.lifecycleSidecarContainer.resources | nindent 12 }}
           env:
             - name: HOST_IP
               valueFrom:

templates/mesh-gateway-deployment.yaml

@@ -90,13 +90,7 @@ spec:
           volumeMounts:
           - name: consul-bin
             mountPath: /consul-bin
-          resources:
-            requests:
-              memory: "25Mi"
-              cpu: "50m"
-            limits:
-              memory: "25Mi"
-              cpu: "50m"
+          resources: {{ toYaml .Values.global.initContainer.resources | nindent 12 }}
         {{- if (and .Values.global.tls.enabled .Values.global.tls.enableAutoEncrypt) }}
         {{- include "consul.getAutoEncryptClientCA" . | nindent 8 }}
         {{- end }}
@@ -330,6 +324,7 @@ spec:
               mountPath: /consul/tls/ca
               readOnly: true
             {{- end }}
+          resources: {{ toYaml .Values.global.lifecycleSidecarContainer.resources | nindent 12 }}
           env:
             - name: HOST_IP
               valueFrom:
@@ -356,13 +351,10 @@ spec:
             {{- if .Values.global.acls.manageSystemACLs }}
             - -token-file=/consul/service/acl-token
             {{- end }}
+          {{- if ( default .Values.global.lifecycleSidecarContainerResources .resources) }}
           resources:
-            requests:
-              memory: "25Mi"
-              cpu: "10m"
-            limits:
-              memory: "25Mi"
-              cpu: "10m"
+            {{ toYaml (default .Values.global.lifecycleSidecarContainerResources .resources) | nindent 12 }}
+          {{- end }}
       {{- if .Values.meshGateway.priorityClassName }}
       priorityClassName: {{ .Values.meshGateway.priorityClassName | quote }}
       {{- end }}

templates/terminating-gateways-deployment.yaml

@@ -124,6 +124,7 @@ spec:
           volumeMounts:
           - name: consul-bin
             mountPath: /consul-bin
+          resources: {{ toYaml $root.Values.global.initContainer.resources | nindent 12 }}
         {{- if (and $root.Values.global.tls.enabled $root.Values.global.tls.enableAutoEncrypt) }}
         {{- include "consul.getAutoEncryptClientCA" $root | nindent 8 }}
         {{- end }}
@@ -319,6 +320,7 @@ spec:
               mountPath: /consul/tls/ca
               readOnly: true
             {{- end }}
+          resources: {{ toYaml $root.Values.global.lifecycleSidecarContainer.resources | nindent 12 }}
           env:
             - name: HOST_IP
               valueFrom:

test/unit/connect-inject-deployment.bats

@@ -920,6 +920,104 @@ load _helpers
   [ "${actual}" = '{"limits":{"cpu":"200m","memory":"200Mi"},"requests":{"cpu":"100m","memory":"100Mi"}}' ]
 }
 
+@test "connectInject/Deployment: default init and sidecar container resources" {
+  cd `chart_dir`
+  local cmd=$(helm template \
+      -s templates/connect-inject-deployment.yaml \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-memory-request=25Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-cpu-request=50m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-memory-limit=125Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-cpu-limit=50m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-memory-request=25Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-cpu-request=20m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-memory-limit=25Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-cpu-limit=20m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+@test "connectInject/Deployment: can set init container resources" {
+  cd `chart_dir`
+  local cmd=$(helm template \
+      -s templates/connect-inject-deployment.yaml \
+      --set 'connectInject.enabled=true' \
+      --set 'global.initContainer.resources.requests.memory=100Mi' \
+      --set 'global.initContainer.resources.requests.cpu=100m' \
+      --set 'global.initContainer.resources.limits.memory=200Mi' \
+      --set 'global.initContainer.resources.limits.cpu=200m' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-memory-request=100Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-cpu-request=100m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-memory-limit=200Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-init-container-cpu-limit=200m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "connectInject/Deployment: lifecycle sidecar container resources" {
+  cd `chart_dir`
+  local cmd=$(helm template \
+      -s templates/connect-inject-deployment.yaml \
+      --set 'connectInject.enabled=true' \
+      --set 'global.lifecycleSidecarContainer.resources.requests.memory=100Mi' \
+      --set 'global.lifecycleSidecarContainer.resources.requests.cpu=100m' \
+      --set 'global.lifecycleSidecarContainer.resources.limits.memory=200Mi' \
+      --set 'global.lifecycleSidecarContainer.resources.limits.cpu=200m' \
+      . | tee /dev/stderr |
+      yq '.spec.template.spec.containers[0].command' | tee /dev/stderr)
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-memory-request=100Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-cpu-request=100m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-memory-limit=200Mi"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+
+  local actual=$(echo "$cmd" |
+    yq 'any(contains("-lifecycle-sidecar-cpu-limit=200m"))' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
 #--------------------------------------------------------------------
 # sidecarProxy.resources
 

test/unit/ingress-gateways-deployment.bats

@@ -446,6 +446,36 @@ load _helpers
   [ "${actual}" = "gwcpu2" ]
 }
 
+@test "ingressGateways/Deployment: init container has default resources" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.spec.initContainers[0].resources' | tee /dev/stderr)
+
+  [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.requests.cpu') = "50m" ]
+  [ $(echo "${actual}" | yq -r '.limits.memory') = "125Mi" ]
+  [ $(echo "${actual}" | yq -r '.limits.cpu') = "50m" ]
+}
+
+@test "ingressGateways/Deployment: lifecycle sidecar has default resources" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/ingress-gateways-deployment.yaml  \
+      --set 'ingressGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.spec.containers[1].resources' | tee /dev/stderr)
+
+  [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.requests.cpu') = "20m" ]
+  [ $(echo "${actual}" | yq -r '.limits.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.limits.cpu') = "20m" ]
+}
+
 #--------------------------------------------------------------------
 # affinity
 

test/unit/mesh-gateway-deployment.bats

@@ -295,6 +295,36 @@ key2: value2' \
   [ "${actual}" = "bar" ]
 }
 
+@test "meshGateway/Deployment: init container has default resources" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.spec.initContainers[0].resources' | tee /dev/stderr)
+
+  [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.requests.cpu') = "50m" ]
+  [ $(echo "${actual}" | yq -r '.limits.memory') = "125Mi" ]
+  [ $(echo "${actual}" | yq -r '.limits.cpu') = "50m" ]
+}
+
+@test "meshGateway/Deployment: lifecycle sidecar has default resources" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/mesh-gateway-deployment.yaml  \
+      --set 'meshGateway.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.spec.containers[1].resources' | tee /dev/stderr)
+
+  [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.requests.cpu') = "20m" ]
+  [ $(echo "${actual}" | yq -r '.limits.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.limits.cpu') = "20m" ]
+}
+
 #--------------------------------------------------------------------
 # containerPort
 

test/unit/terminating-gateways-deployment.bats

@@ -506,6 +506,36 @@ load _helpers
   [ "${actual}" = "gwcpu2" ]
 }
 
+@test "terminatingGateways/Deployment: init container has default resources" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.spec.initContainers[0].resources' | tee /dev/stderr)
+
+  [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.requests.cpu') = "50m" ]
+  [ $(echo "${actual}" | yq -r '.limits.memory') = "125Mi" ]
+  [ $(echo "${actual}" | yq -r '.limits.cpu') = "50m" ]
+}
+
+@test "terminatingGateways/Deployment: lifecycle sidecar has default resources" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      -s templates/terminating-gateways-deployment.yaml  \
+      --set 'terminatingGateways.enabled=true' \
+      --set 'connectInject.enabled=true' \
+      . | tee /dev/stderr |
+      yq -s -r '.[0].spec.template.spec.containers[1].resources' | tee /dev/stderr)
+
+  [ $(echo "${actual}" | yq -r '.requests.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.requests.cpu') = "20m" ]
+  [ $(echo "${actual}" | yq -r '.limits.memory') = "25Mi" ]
+  [ $(echo "${actual}" | yq -r '.limits.cpu') = "20m" ]
+}
+
 #--------------------------------------------------------------------
 # affinity
 

values.yaml

@@ -219,6 +219,32 @@ global:
     # Requires consul-k8s 0.15.0+.
     createFederationSecret: false
 
+  # Resource settings for lifecycle-sidecar containers.
+  # The lifecycle sidecar ensures the Consul services are always registered with
+  # their local consul clients and is used by the ingress/terminating/mesh gateways
+  # as well as with every connect-injected service.
+  lifecycleSidecarContainer:
+    resources:
+      requests:
+        memory: "25Mi"
+        cpu: "20m"
+      limits:
+        memory: "25Mi"
+        cpu: "20m"
+
+  # Resource settings for copy-consul-bin init containers which are used by the
+  # ingress/terminating/mesh gateways.
+  # These settings are bounded by the size of the consul binary
+  # as we use `cp` to copy it into a shared volume in the init container.
+  initContainer:
+    resources:
+      requests:
+        memory: "25Mi"
+        cpu: "50m"
+      limits:
+        memory: "125Mi"
+        cpu: "50m"
+
 # Server, when enabled, configures a server cluster to run. This should
 # be disabled if you plan on connecting to a Consul cluster external to
 # the Kube cluster.