From 7479cda720416daa1db80f56abcadd894481e55d Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Wed, 1 Nov 2023 18:48:04 -0700
Subject: [PATCH 01/24] chore: add TODOs for split

---
 charts/kong/values.yaml | 191 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 190 insertions(+), 1 deletion(-)

diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 115c73b2a..7d07accdf 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -13,7 +13,98 @@
 # Deployment parameters
 # -----------------------------------------------------------------------------
 
+# TODO 921 we added this key to hold Deployment-level settings, though it does
+# not contain all of them. Many settings (e.g. `.deploymentAnnotations`) were
+# created at the root earlier in the chart's life, before we thought to
+# organize them by resource better.
+#
+# The same is true of Pod settings, which naturally nest under Deployment
+# settings, but were often created at the root (e.g. `.env`).
+#
+# We already have a `kong` key here, so adding a `controller` key follows
+# pretty naturally for a separate controller Deployment. Moving the current
+# contents of `.ingressController` under here would be simplest for migration,
+# but would keep some other bad organization.
+#
+# The keys under `ingressController` are a mixture of container-level (e.g.
+# `readinessProbe`), application-level (e.g. `gatewayDiscovery`), and external
+# resource-level (e.g. `admissionWebhook`) settings. The former two can
+# reasonably enough be grouped under here, but the external resources arguably
+# should not be. For maximum reorganization, I'd recommend something like:
+# 
+# ```
+# deployment:
+#   controller:
+#     annotations: {} # Deployment-level annotations
+#     autoscaling: {} # A duplicate of the current root key, for the controller Deployment. This may actually be a separate autoscaling.controller section.
+#     pod:
+#       annotations: {} # Pod-level annotations. currently there is a single root-level .podAnnotations used for the single unified Pod
+#       serviceAccount: {}
+#       container:
+#         env: {...} # controller environment
+#         image: {}
+#       sidecarContainers: [] # attach a tcpdump or what have you
+#   kong: {} # same as the controller section
+#
+# admissionWebhook: {} # the current contents for .ingressController.admissionWebhook
+# ```
+#
+# A more conservative approach is to preserve existing settings where
+# available. This means that existing configuration doesn't need to be
+# modified, but will be incongruous. You would have, for example,
+# `.deployment.controller.pod.annotations` to set Pod annotations for the
+# controller Deployment, but `.podAnnotations` for the Kong Deployment's Pods.
 deployment:
+  controller:
+    annotations: {} # TODO 921 add any defaults
+    # TODO 921 the equivalent root setting has "default" values: https://github.com/Kong/charts/blob/kong-2.30.0/charts/kong/values.yaml#L887-L904
+    # These are no-oped with an `enabled=false` key, but are somewhat
+    # misleading in that we didn't really design them for any particular
+    # purpose--they're just values to show the structure. Given how values.yaml
+    # defaults work IDK if maybe we'd consider making the _actual_ values.yaml
+    # base not set defaults in these cases, and then having a dedicated example
+    # values.yaml that just shows structure.
+    autoscaling: {}
+    # TODO 921 parity, though it really makes _no_ sense for a daemonset
+    # controller. that said, we may be able to use the same templates for both
+    # Deployments' basic Deployment-y things and get it for free.
+    daemonset: false
+    pod:
+      annotations: {} # .podAnnotations, for the controller
+      serviceAccount:
+        create: true
+        # TODO 921 we arguably don't need this with a split controller
+        # Deployment, since we use it to hide the token from Kong and its
+        # arbitrary plugin codeTODO 921 we arguably don't need this with a
+        # split controller Deployment, since we use it to hide the token from
+        # Kong and its arbitrary plugin code.
+        automountServiceAccountToken: false
+      container:
+        env:
+          # The controller disables TLS verification by default because Kong
+          # generates self-signed certificates by default. Set this to false once you
+          # have installed CA-signed certificates.
+          kong_admin_tls_skip_verify: true
+          # If using Kong Enterprise with RBAC enabled, uncomment the section below
+          # and specify the secret/key containing your admin token.
+          # kong_admin_token:
+          #   valueFrom:
+          #     secretKeyRef:
+          #        name: CHANGEME-admin-token-secret
+          #        key: CHANGEME-admin-token-key
+        image:
+          repository: kong/kubernetes-ingress-controller
+          tag: "2.12"
+          # Optionally set a semantic version for version-gated features. This can normally
+          # be left unset. You only need to set this if your tag is not a semver string,
+          # such as when you are using a "next" tag. Set this to the effective semantic
+          # version of your tag: for example if using a "next" image for an unreleased 3.1.0
+          # version, set this to "3.1.0".
+          effectiveSemver:
+        userDefinedVolumeMounts: []
+      sidecarContainers: []
+      userDefinedVolumes: []
+      initContainers: []
   kong:
     # Enable or disable Kong itself
     # Setting this to false with ingressController.enabled=true will create a
@@ -22,6 +113,7 @@ deployment:
   ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing,
   ## for it to be considered available.
   # minReadySeconds: 60
+  # TODO 921 Kong presumably no longer needs a service account at all.
   ## Specify the service account to create and to be assigned to the deployment / daemonset and for the migrations
   serviceAccount:
     create: true
@@ -51,6 +143,7 @@ deployment:
   # userDefinedVolumes:
   # - name: "volumeName"
   #   emptyDir: {}
+  # TODO 921 these would live inside the container section
   # userDefinedVolumeMounts:
   # - name: "volumeName"
   #   mountPath: "/opt/user/dir/mount"
@@ -60,6 +153,9 @@ deployment:
   # Use a DaemonSet controller instead of a Deployment controller
   daemonset: false
   hostNetwork: false
+  # TODO 921 part of the container section. this is kong-specific functionality
+  # though--dunno if we want to handle stuff that isn't generic to either 
+  # Deployment's container differently than stuff that does
   # kong_prefix empty dir size
   prefixDir:
     sizeLimit: 256Mi
@@ -82,6 +178,7 @@ deployment:
 # Values here take precedence over values from other sections of values.yaml,
 # e.g. setting pg_user here will override the value normally set when postgresql.enabled
 # is set below. In general, you should not set values here if they are set elsewhere.
+# TODO 921 deployments.kong.pod.container.env
 env:
   database: "off"
   # the chart uses the traditional router (for Kong 3.x+) because the ingress
@@ -107,6 +204,7 @@ env:
 # Note that these environment variables will only apply to the proxy and init container. The ingress-controller
 # container has its own customEnv section.
 
+# TODO 921 deployments.kong.pod.container.customEnv
 # customEnv:
 #   api_token:
 #     valueFrom:
@@ -118,6 +216,7 @@ env:
 # This section can be used to configure some extra labels that will be added to each Kubernetes object generated.
 extraLabels: {}
 
+# TODO 921 deployments.kong.pod.container.image
 # Specify Kong's Docker image and repository details here
 image:
   repository: kong
@@ -136,6 +235,14 @@ image:
   # pullSecrets:
   #   - myRegistrKeySecretName
 
+# TODO 921 we can probably leave the service definitions at the root. Although
+# they're specific to a particular Deployment, they can't be defined
+# arbitrarily (you can only configure admin, proxy, etc. that we've given you)
+# and we know what Deployment they target. Controller-related Services AFAIK
+# have no such configuration (we create a webhook service, but it's done
+# through the webhook config, not through anything that uses the standard
+# service template.
+#
 # Specify Kong admin API service and listener configuration
 admin:
   # Enable creating a Kubernetes service for the admin API
@@ -415,6 +522,10 @@ udpProxy:
     #   # "ssl" is required for SNI-based routes. It is not supported on versions <2.0
     #   parameters: []
 
+# TODO 921 these go into the Kong deployment but probably don't need to move
+# there's no controller equivalent we'd ever need controller/kong separate
+# config.
+#
 # Custom Kong plugins can be loaded into Kong by mounting the plugin code
 # into the file-system of Kong container.
 # The plugin code should be present in ConfigMap or Secret inside the same
@@ -435,6 +546,22 @@ plugins: {}
   # secrets:
   # - pluginName: rewriter
   #   name: kong-plugin-rewriter
+
+# TODO 921 We could maybe convert these into standard user mounts? a basic
+# rearrange migration doesn't work, but you can basically handle these by
+# creating a defined volume that fills in the automatic fields:
+#
+# userDefinedVolumes:
+# - name: "<secret-name>"
+#   secret:
+#     name: whatever
+# userDefinedVolumeMounts:
+# - name: "<secret-name>"
+#   mountPath: "/etc/secrets/secret-name"
+#
+# if we leave it as-is, probably no reason to port it to the controller
+# deployment--we can just require creating the full mount config there.
+
 # Inject specified secrets as a volume in Kong Container at path /etc/secrets/{secret-name}/
 # This can be used to override default SSL certificates.
 # Be aware that the secret name will be used verbatim, and that certain types
@@ -447,10 +574,18 @@ secretVolumes: []
 
 # Enable/disable migration jobs, and set annotations for them
 migrations:
+  # TODO 921 the new pattern would have individual configurations for each
+  # migration, which is probably more annoying than useful. there shouldn't be
+  # a reason to assign, say, more resources to post migrations.
   # Enable pre-upgrade migrations (run "kong migrations up")
   preUpgrade: true
   # Enable post-upgrade migrations (run "kong migrations finish")
   postUpgrade: true
+  # TODO 921 these do not follow the pattern, but it's probably less crucial
+  # that they do. our allowed configuration for migrations is pretty limited.
+  # we may want to just do it anyway--once you're using a migration tool,
+  # there's not a whole lot of reason to be conservative. the work required is
+  # all on the dev side, and adding more key remaps is easy.
   # Annotations to apply to migrations job pods
   # By default, these disable service mesh sidecar injection for Istio and Kuma,
   # as the sidecar containers do not terminate and prevent the jobs from completing
@@ -462,6 +597,7 @@ migrations:
   jobAnnotations: {}
   # Optionally set a backoffLimit. If none is set, Jobs will use the cluster default
   backoffLimit:
+  # TODO 921 if we do move stuff here, this is part of the container
   resources: {}
   # Example reasonable setting for "resources":
   # resources:
@@ -512,8 +648,10 @@ dblessConfig:
 # Kong Ingress Controller's primary purpose is to satisfy Ingress resources
 # created in k8s.  It uses CRDs for more fine grained control over routing and
 # for Kong specific configuration.
+# TODO 921 deployments.controller.enabled
 ingressController:
   enabled: true
+  # TODO 921 deployments.controller.pod.container.image
   image:
     repository: kong/kubernetes-ingress-controller
     tag: "2.12"
@@ -523,8 +661,13 @@ ingressController:
     # version of your tag: for example if using a "next" image for an unreleased 3.1.0
     # version, set this to "3.1.0".
     effectiveSemver:
+  # TODO 921 deployments.controller.pod.container.args
   args: []
 
+  # TODO 921 probably goes under a root-level controller field, which is, well,
+  # where it is now. this is config for the controller _not_ related to its
+  # kubernetes resources, so it doesn't necessarily fit under the `deployment`
+  # section.
   gatewayDiscovery:
     enabled: false
     generateAdminApiService: false
@@ -532,6 +675,7 @@ ingressController:
       namespace: ""
       name: ""
 
+  # TODO 921 app-level config, probably remains here
   # Specify individual namespaces to watch for ingress configuration. By default,
   # when no namespaces are set, the controller watches all namespaces and uses a
   # ClusterRole to grant access to Kubernetes resources. When you list specific
@@ -542,6 +686,7 @@ ingressController:
   watchNamespaces: []
 
   # Specify Kong Ingress Controller configuration via environment variables
+  # TODO 921 deployments.controller.pod.container.env
   env:
     # The controller disables TLS verification by default because Kong
     # generates self-signed certificates by default. Set this to false once you
@@ -560,6 +705,8 @@ ingressController:
   # customEnv:
   #   TZ: "Europe/Berlin"
 
+  # TODO 921 separate resource config, probably remains here. only loosely-tied
+  # to the deployment through a service we generate for you.
   admissionWebhook:
     enabled: true
     failurePolicy: Ignore
@@ -579,6 +726,7 @@ ingressController:
     # Tune the default Kubernetes timeoutSeconds of 10 seconds
     # timeoutSeconds: 10
 
+  # TODO 921 app-level config, probably remains here
   ingressClass: kong
   # annotations for IngressClass resource (Kubernetes 1.18+)
   ingressClassAnnotations: {}
@@ -589,11 +737,13 @@ ingressController:
   # - name: "volumeName"
   #   mountPath: "/opt/user/dir/mount"
 
+  # TODO 921 separate resource config, probably remains here
   rbac:
     # Specifies whether RBAC resources should be created
     create: true
 
   # general properties
+  # TODO 921 deployments.controller.pod.container.livenessProbe
   livenessProbe:
     httpGet:
       path: "/healthz"
@@ -604,6 +754,7 @@ ingressController:
     periodSeconds: 10
     successThreshold: 1
     failureThreshold: 3
+  # TODO 921 deployments.controller.pod.container.readinessProbe
   readinessProbe:
     httpGet:
       path: "/readyz"
@@ -614,6 +765,7 @@ ingressController:
     periodSeconds: 10
     successThreshold: 1
     failureThreshold: 3
+  # TODO 921 deployments.controller.pod.container.resources
   resources: {}
   # Example reasonable setting for "resources":
   # resources:
@@ -624,6 +776,7 @@ ingressController:
   #     cpu: 50m
   #     memory: 128Mi
 
+  # TODO 921 app-level config, probably remains here
   konnect:
     enabled: false
 
@@ -643,6 +796,7 @@ ingressController:
       # Specifies whether the controller should fetch a license from Konnect and apply it to managed Gateways.
       enabled: false
 
+  # TODO 921 app-level config, probably remains here
   adminApi:
     tls:
       client:
@@ -770,6 +924,8 @@ certificates:
 # Miscellaneous parameters
 # -----------------------------------------------------------------------------
 
+# TODO 921 kinda part of the kong deployment, but not generic deployment
+# functionality.
 waitImage:
   # Wait for the database to come online before starting Kong or running migrations
   # If Kong is to access the database through a service mesh that injects a sidecar to
@@ -784,6 +940,7 @@ waitImage:
   # tag: 5
   pullPolicy: IfNotPresent
 
+# TODO 921 deployment.*.updateStrategy
 # update strategy
 updateStrategy: {}
   # type: RollingUpdate
@@ -791,6 +948,7 @@ updateStrategy: {}
   #   maxSurge: "100%"
   #   maxUnavailable: "0%"
 
+# TODO 921 deployments.kong.pod.container.resources
 # If you want to specify resources, uncomment the following
 # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
 resources: {}
@@ -801,6 +959,7 @@ resources: {}
   #  cpu: 1
   #  memory: 2G
 
+# TODO 921 deployments.kong.pod.container.readinessProbe
 # readinessProbe for Kong pods
 readinessProbe:
   httpGet:
@@ -813,6 +972,7 @@ readinessProbe:
   successThreshold: 1
   failureThreshold: 3
 
+# TODO 921 deployments.kong.pod.container.livenessProbe
 # livenessProbe for Kong pods
 livenessProbe:
   httpGet:
@@ -825,6 +985,7 @@ livenessProbe:
   successThreshold: 1
   failureThreshold: 3
 
+# TODO 921 deployments.kong.pod.container.startupProbe
 # startupProbe for Kong pods
 # startupProbe:
 #   httpGet:
@@ -837,6 +998,7 @@ livenessProbe:
 #   successThreshold: 1
 #   failureThreshold: 40
 
+# TODO 921 deployments.kong.pod.container.lifecycle
 # Proxy container lifecycle hooks
 # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
 lifecycle:
@@ -849,41 +1011,54 @@ lifecycle:
         - quit
         - '--wait=15'
 
+# TODO 921 deployments.kong.pod.terminationGracePeriodSeconds
 # Sets the termination grace period for pods spawned by the Kubernetes Deployment.
 # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution
 terminationGracePeriodSeconds: 30
 
+# TODO 921 deployments.kong.pod.affinity
 # Affinity for pod assignment
 # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
 # affinity: {}
 
+# TODO 921 deployments.kong.pod.topologySpreadConstraints
 # Topology spread constraints for pod assignment (requires Kubernetes >= 1.19)
 # Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
 # topologySpreadConstraints: []
 
+# TODO 921 deployments.kong.pod.tolerations
 # Tolerations for pod assignment
 # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
 tolerations: []
 
+# TODO 921 either part of the pod or deployment (or under affinity?), not sure
+# need to look up
 # Node labels for pod assignment
 # Ref: https://kubernetes.io/docs/user-guide/node-selection/
 nodeSelector: {}
 
+# TODO 921 deployments.kong.pod.annotations
 # Annotation to be added to Kong pods
 podAnnotations:
   kuma.io/gateway: enabled
   traffic.sidecar.istio.io/includeInboundPorts: ""
 
+# TODO 921 deployments.kong.pod.labels
 # Labels to be added to Kong pods
 podLabels: {}
 
+# TODO 921 deployments.kong.replicaCount
 # Kong pod count.
 # It has no effect when autoscaling.enabled is set to true
 replicaCount: 1
 
+# TODO 921 deployments.kong.annotations
 # Annotations to be added to Kong deployment
 deploymentAnnotations: {}
 
+# TODO 921 this is a resource separate from the Deployment, but attached to it
+# not sure if we should nest under deployment config or create
+# `autoscaling.kong` and `autoscaling.controller` keys.
 # Enable autoscaling using HorizontalPodAutoscaler
 # When configuring an HPA, you must set resource requests on all containers via
 # "resources" and, if using the controller, "ingressController.resources" in values.yaml
@@ -903,6 +1078,7 @@ autoscaling:
           type: Utilization
           averageUtilization: 80
 
+# TODO 921 ditto autoscaling
 # Kong Pod Disruption Budget
 podDisruptionBudget:
   enabled: false
@@ -910,6 +1086,9 @@ podDisruptionBudget:
   # maxUnavailable: "50%"
   # minAvailable: "50%"
 
+# TODO 921 discard this? unrelated to the task mostly since PSP is separate,
+# but it's basically removed at this point. need to confirm what the minimum
+# Pod security standards version is instead
 podSecurityPolicy:
   enabled: false
   labels: {}
@@ -939,12 +1118,14 @@ podSecurityPolicy:
     # If you use Kong Enterprise <1.5, this must be set to false.
     readOnlyRootFilesystem: true
 
-
+# TODO 921 figure out what this is and where it should go
 priorityClassName: ""
 
+# TODO 921 deployments.*.pod.securityContext
 # securityContext for Kong pods.
 securityContext: {}
 
+# TODO 921 deployments.*.pod.container.securityContext
 # securityContext for containers.
 containerSecurityContext:
   readOnlyRootFilesystem: true
@@ -957,6 +1138,7 @@ containerSecurityContext:
     drop:
     - ALL
 
+# TODO 921 deployments.*.pod.dnsPolicy probably. confirm its actual location
 ## Optional DNS configuration for Kong pods
 # dnsPolicy: ClusterFirst
 # dnsConfig:
@@ -971,6 +1153,7 @@ containerSecurityContext:
 #   - cluster.local
 #   - us-east-1.compute.internal
 
+# TODO 921 figure out if anything is necessary for this
 serviceMonitor:
   # Specifies whether ServiceMonitor for Prometheus operator should be created
   # If you wish to gather metrics from a Kong instance with the proxy disabled (such as a hybrid control plane), see:
@@ -991,6 +1174,10 @@ serviceMonitor:
 # Kong Enterprise parameters
 # -----------------------------------------------------------------------------
 
+# TODO 921 the `enterprise` section is Kong-specific application-level
+# configurtion. As such it does not really map to any location in a Kubernetes
+# resource, so it probably just remains here at the root.
+
 # Toggle Kong Enterprise features on or off
 # RBAC and SMTP configuration have additional options that must all be set together
 # Other settings should be added to the "env" settings below
@@ -1213,6 +1400,8 @@ clustertelemetry:
     # Each path in an Ingress is required to have a corresponding path type. (ImplementationSpecific/Exact/Prefix)
     pathType: ImplementationSpecific
 
+# TODO 921 maybe keep these as-is but do not add them to the generic handler.
+# They are also features that are special cases of userDefinedVolume(Mount)s
 extraConfigMaps: []
 # extraConfigMaps:
 # - name: my-config-map

From bc2610c08a909801dfae2b3144eae2ee7f759a9b Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Thu, 2 Nov 2023 16:46:33 -0700
Subject: [PATCH 02/24] wip: commentary around template redesign

---
 charts/kong/templates/_helpers.tpl            |  82 +++
 .../kong/templates/deployment-controller.yaml | 638 ++++++++++++++++++
 charts/kong/values.yaml                       |   4 +-
 3 files changed, 723 insertions(+), 1 deletion(-)
 create mode 100644 charts/kong/templates/deployment-controller.yaml

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index 26c91ae74..ee7777ab3 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -460,6 +460,10 @@ The name of the Service which will be used by the controller to update the Ingre
 {{- (printf "%s/%s" ( include "kong.namespace" . ) ( default ( printf "%s-proxy" (include "kong.fullname" . )) $proxyOverride )) -}}
 {{- end -}}
 
+# TODO 921 everything here would need to become relative. some of it relies on
+# external values though, e.g. .Values.ingressController.ingressClass and
+# .Values.ingressController.admissionWebhook.address presumably remain where
+# they are.
 {{- define "kong.ingressController.env" -}}
 {{/*
     ====== AUTO-GENERATED ENVIRONMENT VARIABLES ======
@@ -919,6 +923,84 @@ The name of the Service which will be used by the controller to update the Ingre
   {{- include "controller.adminApiCertVolumeMount" . | nindent 2 }}
 {{- end -}}
 
+{{- define "kong.controller-container-new" -}}
+- name: ingress-controller
+  securityContext:
+{{ toYaml .securityContext | nindent 4 }}
+  args:
+  {{ if .args}}
+  {{- range $val := .args }}
+  - {{ $val }}
+  {{- end }}
+  {{- end }}
+  ports:
+  # TODO 921 would not have global value here, need to pass something in. can
+  # pass in port or null if none--it's the only value here
+  {{- if .Values.ingressController.admissionWebhook.enabled }}
+  - name: webhook
+    containerPort: {{ .Values.ingressController.admissionWebhook.port }}
+    protocol: TCP
+  {{- end }}
+  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .image)) -}}
+  - name: cmetrics
+    containerPort: 10255
+    protocol: TCP
+  {{- end }}
+  env:
+  - name: POD_NAME
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.name
+  - name: POD_NAMESPACE
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.namespace
+# TODO 921 this is not currently designed to be at all relative and relies on
+# external sections
+{{- include "kong.ingressController.env" .  | indent 2 }}
+  image: {{ include "kong.getRepoTag" .image }}
+  imagePullPolicy: {{ .image.pullPolicy }}
+{{/* disableReadiness is a hidden setting to drop this block entirely for use with a debugger
+     Helm value interpretation doesn't let you replace the default HTTP checks with any other
+     check type, and all HTTP checks freeze when a debugger pauses operation.
+     Setting disableReadiness to ANY value disables the probes.
+*/}}
+{{- if (not (hasKey .Values.ingressController "disableProbes")) }}
+  readinessProbe:
+{{ toYaml .readinessProbe | indent 4 }}
+  livenessProbe:
+{{ toYaml .livenessProbe | indent 4 }}
+{{- end }}
+  resources:
+{{ toYaml .resources | indent 4 }}
+  volumeMounts:
+# TODO 921 comes from external, but only needs the single boolean in the
+# initial if
+{{- if .ingressController.admissionWebhook.enabled }}
+  - name: webhook-cert
+    mountPath: /admission-webhook
+    readOnly: true
+{{- end }}
+# TODO 921 would not be in the container context
+{{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+  - name: {{ template "kong.serviceAccountTokenName" . }}
+    mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+    readOnly: true
+{{- end }}
+  {{- include "kong.userDefinedVolumeMounts" . | nindent 2 }}
+  # TODO 921 relies on some content under .ingressController
+  {{- include "controller.adminApiCertVolumeMount" . | nindent 2 }}
+{{- end -}}
+
+{{- define "secretkeyref" -}}
+valueFrom:
+  secretKeyRef:
+    name: {{ .name }}
+    key: {{ .key }}
+{{- end -}}
+
 {{- define "secretkeyref" -}}
 valueFrom:
   secretKeyRef:
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
new file mode 100644
index 000000000..b81ed54ec
--- /dev/null
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -0,0 +1,638 @@
+{{- define "kong.deployment" -}}
+apiVersion: apps/v1
+{{- if .daemonset }}
+kind: DaemonSet
+{{- else }}
+kind: Deployment
+{{- end }}
+metadata:
+  name: {{ template "kong.fullname" . }} # TODO 921 needs suffix for deployment via arg. also needs to be passed in
+  namespace:  {{ template "kong.namespace" . }}
+  labels:
+    {{- include "kong.metaLabels" . | nindent 4 }} # TODO 921 needs to be passed in for root
+    app.kubernetes.io/component: app
+  {{- if .annotations }}
+  annotations:
+  {{- range $key, $value := .annotations }}
+    {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
+  {{- end }}
+  {{- end }}
+spec:
+  {{- if not .autoscaling.enabled }}
+  {{- if not .daemonset }}
+  replicas: {{ .replicaCount }}
+  {{- end }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "kong.selectorLabels" . | nindent 6 }} # TODO 921 needs to be passed in
+  {{- if .updateStrategy }}
+  {{- if .daemonset }}
+  updateStrategy:
+  {{- else }}
+  strategy:
+  {{- end }}
+{{ toYaml .Values.updateStrategy | indent 4 }}
+  {{- end }}
+  {{- if .minReadySeconds }}
+  minReadySeconds: {{ .minReadySeconds }}
+  {{- end }}
+
+  template:
+    metadata:
+      annotations:
+        {{/* # TODO 921 some keys in the pod template end up being from the deployment dict, so probably no separate define 
+             # there's no obvious reason to separate them, since there can only be one per Deployment anyway */}}
+        {{- if (and (not .serviceAccount.automountServiceAccountToken) (or .serviceAccount.create .serviceAccount.name)) }}
+        {{/* # TODO 921 needs to be passed in as-is. possibly should be made relative, since the generator will attempt to use the same
+             # name, but lives in a separate service-account.yaml template, so vov */}}
+        kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
+        {{- end }}
+        {{/*  # TODO 921 special to DB-less Kong with static config. I think this can be handled with the .Values.dblessConfig.config only:
+              # if you do not want to redeploy on hash change, do not provide a static config. However, this would fire in both controller
+              # and proxy calls, so we probably need to have injection for additional Pod (and whatever else) annotations.
+        */}}
+        {{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off" )) }}
+        {{- if .Values.dblessConfig.config }}
+        checksum/dbless.config: {{ toYaml .Values.dblessConfig.config | sha256sum }}
+        {{- end }}
+        {{- end }}
+        {{/* # TODO 921 end weird config hash block */}}
+        {{- if .pod.annotations }}
+        {{- range $key, $value := .pod.annotations }}
+        {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
+        {{- end }}
+        {{- end }}
+      labels:
+        {{- include "kong.metaLabels" . | nindent 8 }} # TODO 921 passed in
+        app.kubernetes.io/component: app
+        app: {{ template "kong.fullname" . }} # TODO 921 passed in, needs suffix
+        version: {{ .Chart.AppVersion | quote }} # TODO 921 passed in
+        {{- if .pod.labels }}
+        {{ include "kong.renderTpl" (dict "value" .pod.labels "context" $) | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- if .hostNetwork }}
+      hostNetwork: true
+      {{- end }}
+      {{- if .pod.priorityClassName }}
+      priorityClassName: "{{ .pod.priorityClassName }}"
+      {{- end }}
+      {{- if or .serviceAccount.create .serviceAccount.name }}
+      serviceAccountName: {{ template "kong.serviceAccountName" . }} # TODO 921 passed in
+      {{- end }}
+      {{- if (and (or .serviceAccount.create .serviceAccount.name) .serviceAccount.automountServiceAccountToken) }}
+      automountServiceAccountToken: true
+      {{- else }}
+      automountServiceAccountToken: false
+      {{ end }}
+      {{- if .pod.container.image.pullSecrets }}
+      # TODO 921 this is out of place, but the rest of .image _does_ live
+      # inside the container, so probably just keep it there, rather than
+      # separating into .pod.imagePullSecrets
+      imagePullSecrets:
+      {{- range .pod.container.image.pullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      # TODO 921 this is Kong-specific and does not rely on any settings in the
+      # deployment config, it's just a static-ish initcontainer we inject. it
+      # does need Kong's image, security context, resources, env, so probably
+      # create a dedicated helper to create the initContainer and then support
+      # injecting initContainers in general. .Values stuff left as-is from here
+      {{- if .Values.deployment.kong.enabled }}
+      initContainers:
+      - name: clear-stale-pid
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        command:
+        - "rm"
+        - "-vrf"
+        - "$KONG_PREFIX/pids"
+        env:
+        {{- include "kong.env" . | nindent 8 }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 8 }}
+        {{- if .Values.deployment.initContainers }}
+        {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+        {{- end }}
+        {{- if (and (not (eq .Values.env.database "off")) .Values.waitImage.enabled) }}
+        {{- include "kong.wait-for-db" . | nindent 6 }}
+        {{- end }}
+      {{- end }} # TODO 921 END initContainers clear-stale-pid
+      {{- if .pod.hostAliases }}
+      hostAliases:
+        {{- toYaml .pod.hostAliases | nindent 6 }}
+      {{- end}}
+      {{- if .dnsPolicy }}
+      dnsPolicy: {{ .pod.dnsPolicy | quote }}
+      {{- end }}
+      {{- if .pod.dnsConfig }}
+      dnsConfig:
+{{ toYaml .pod.dnsConfig | indent 8 }}
+      {{- end }}
+      containers:
+      # TODO 921 all sorts of fun stuff in this section. .pod.container
+      # contains generic configuration, but in practice this needs to be
+      # combined with a bunch of app-specific config that lives elsewhere, and
+      # not just in env. a generic function in context is probably quite
+      # difficult, though doable if you provide pass-in points for everything
+      # (mounts, env, god knows what else) and assign a var at the beginning.
+      # probably just use dedicated functions for kong and controller
+      # containers for now, with a TODO to genericize further
+      {{- if .Values.ingressController.enabled }}
+      {{- include "kong.controller-container" . | nindent 6 }}
+      {{ end }}
+      {{- if .Values.deployment.sidecarContainers }}
+      {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
+      {{- end }}
+      {{- if .Values.deployment.kong.enabled }}
+      - name: "proxy"
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        env:
+        {{- include "kong.no_daemon_env" . | nindent 8 }}
+        lifecycle:
+          {{- toYaml .Values.lifecycle | nindent 10 }}
+        ports:
+        {{- if (and .Values.admin.http.enabled .Values.admin.enabled) }}
+        - name: admin
+          containerPort: {{ .Values.admin.http.containerPort }}
+          {{- if .Values.admin.http.hostPort }}
+          hostPort: {{ .Values.admin.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.admin.tls.enabled .Values.admin.enabled) }}
+        - name: admin-tls
+          containerPort: {{ .Values.admin.tls.containerPort }}
+          {{- if .Values.admin.tls.hostPort }}
+          hostPort: {{ .Values.admin.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.http.enabled .Values.proxy.enabled) }}
+        - name: proxy
+          containerPort: {{ .Values.proxy.http.containerPort }}
+          {{- if .Values.proxy.http.hostPort }}
+          hostPort: {{ .Values.proxy.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.tls.enabled .Values.proxy.enabled)}}
+        - name: proxy-tls
+          containerPort: {{ .Values.proxy.tls.containerPort }}
+          {{- if .Values.proxy.tls.hostPort }}
+          hostPort: {{ .Values.proxy.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- range .Values.proxy.stream }}
+        - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- range .Values.udpProxy.stream }}
+        - name: streamudp-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- if (and .Values.status.http.enabled .Values.status.enabled)}}
+        - name: status
+          containerPort: {{ .Values.status.http.containerPort }}
+          {{- if .Values.status.http.hostPort }}
+          hostPort: {{ .Values.status.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.status.tls.enabled .Values.status.enabled) }}
+        - name: status-tls
+          containerPort: {{ .Values.status.tls.containerPort }}
+          {{- if .Values.status.tls.hostPort }}
+          hostPort: {{ .Values.status.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.cluster.tls.enabled .Values.cluster.enabled) }}
+        - name: cluster-tls
+          containerPort: {{ .Values.cluster.tls.containerPort }}
+          {{- if .Values.cluster.tls.hostPort }}
+          hostPort: {{ .Values.cluster.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if .Values.enterprise.enabled }}
+        {{- if (and .Values.manager.http.enabled .Values.manager.enabled) }}
+        - name: manager
+          containerPort: {{ .Values.manager.http.containerPort }}
+          {{- if .Values.manager.http.hostPort }}
+          hostPort: {{ .Values.manager.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.manager.tls.enabled .Values.manager.enabled) }}
+        - name: manager-tls
+          containerPort: {{ .Values.manager.tls.containerPort }}
+          {{- if .Values.manager.tls.hostPort }}
+          hostPort: {{ .Values.manager.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.http.enabled .Values.portal.enabled) }}
+        - name: portal
+          containerPort: {{ .Values.portal.http.containerPort }}
+          {{- if .Values.portal.http.hostPort }}
+          hostPort: {{ .Values.portal.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.tls.enabled .Values.portal.enabled) }}
+        - name: portal-tls
+          containerPort: {{ .Values.portal.tls.containerPort }}
+          {{- if .Values.portal.tls.hostPort }}
+          hostPort: {{ .Values.portal.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.http.enabled .Values.portalapi.enabled) }}
+        - name: portalapi
+          containerPort: {{ .Values.portalapi.http.containerPort }}
+          {{- if .Values.portalapi.http.hostPort }}
+          hostPort: {{ .Values.portalapi.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.tls.enabled .Values.portalapi.enabled) }}
+        - name: portalapi-tls
+          containerPort: {{ .Values.portalapi.tls.containerPort }}
+          {{- if .Values.portalapi.tls.hostPort }}
+          hostPort: {{ .Values.portalapi.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.clustertelemetry.tls.enabled .Values.clustertelemetry.enabled) }}
+        - name: clustert-tls
+          containerPort: {{ .Values.clustertelemetry.tls.containerPort }}
+          {{- if .Values.clustertelemetry.tls.hostPort }}
+          hostPort: {{ .Values.clustertelemetry.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- end }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 10 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 10 }}
+        readinessProbe:
+{{ include "kong.proxy.compatibleReadiness" . | indent 10 }}
+        livenessProbe:
+{{ toYaml .Values.livenessProbe | indent 10 }}
+        {{- if .Values.startupProbe }}
+        startupProbe:
+{{ toYaml .Values.startupProbe | indent 10 }}
+        {{- end }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        {{- end }} {{/* End of Kong container spec */}}
+    {{- if .Values.affinity }}
+      affinity:
+{{ toYaml .Values.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+{{ toYaml .Values.topologySpreadConstraints | indent 8 }}
+    {{- end }}
+      securityContext:
+      {{- include "kong.podsecuritycontext" . | nindent 8 }}
+    {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+    {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+    {{- if .Values.tolerations }}
+      tolerations:
+{{ toYaml .Values.tolerations | indent 8 }}
+    {{- end }}
+      volumes:
+      {{- include "kong.volumes" . | nindent 8 -}}
+      {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
+
+{{- end -}} {{/* end kong.deployment */}}
+
+{{- if .Values.deployment.controller.enabled }}
+apiVersion: apps/v1
+{{- if .Values.deployment.daemonset }}
+kind: DaemonSet
+{{- else }}
+kind: Deployment
+{{- end }}
+metadata:
+  name: {{ template "kong.fullname" . }}
+  namespace:  {{ template "kong.namespace" . }}
+  labels:
+    {{- include "kong.metaLabels" . | nindent 4 }}
+    app.kubernetes.io/component: app
+  {{- if .Values.deploymentAnnotations }}
+  annotations:
+  {{- range $key, $value := .Values.deploymentAnnotations }}
+    {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
+  {{- end }}
+  {{- end }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  {{- if not .Values.deployment.daemonset }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "kong.selectorLabels" . | nindent 6 }}
+  {{- if .Values.updateStrategy }}
+  {{- if .Values.deployment.daemonset }}
+  updateStrategy:
+  {{- else }}
+  strategy:
+  {{- end }}
+{{ toYaml .Values.updateStrategy | indent 4 }}
+  {{- end }}
+  {{- if .Values.deployment.minReadySeconds }}
+  minReadySeconds: {{ .Values.deployment.minReadySeconds }}
+  {{- end }}
+
+  template:
+    metadata:
+      annotations:
+        {{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+        kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
+        {{- end }}
+        {{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off" )) }}
+        {{- if .Values.dblessConfig.config }}
+        checksum/dbless.config: {{ toYaml .Values.dblessConfig.config | sha256sum }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.podAnnotations }}
+        {{- range $key, $value := .Values.podAnnotations }}
+        {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
+        {{- end }}
+        {{- end }}
+      labels:
+        {{- include "kong.metaLabels" . | nindent 8 }}
+        app.kubernetes.io/component: app
+        app: {{ template "kong.fullname" . }}
+        version: {{ .Chart.AppVersion | quote }}
+        {{- if .Values.podLabels }}
+        {{ include "kong.renderTpl" (dict "value" .Values.podLabels "context" $) | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- if .Values.deployment.hostNetwork }}
+      hostNetwork: true
+      {{- end }}
+      {{- if .Values.priorityClassName }}
+      priorityClassName: "{{ .Values.priorityClassName }}"
+      {{- end }}
+      {{- if or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name }}
+      serviceAccountName: {{ template "kong.serviceAccountName" . }}
+      {{- end }}
+      {{- if (and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) .Values.deployment.serviceAccount.automountServiceAccountToken) }}
+      automountServiceAccountToken: true
+      {{- else }}
+      automountServiceAccountToken: false
+      {{ end }}
+      {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.image.pullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      {{- if .Values.deployment.kong.enabled }}
+      initContainers:
+      - name: clear-stale-pid
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        command:
+        - "rm"
+        - "-vrf"
+        - "$KONG_PREFIX/pids"
+        env:
+        {{- include "kong.env" . | nindent 8 }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 8 }}
+        {{- if .Values.deployment.initContainers }}
+        {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+        {{- end }}
+        {{- if (and (not (eq .Values.env.database "off")) .Values.waitImage.enabled) }}
+        {{- include "kong.wait-for-db" . | nindent 6 }}
+        {{- end }}
+      {{- end }}
+      {{- if .Values.deployment.hostAliases }}
+      hostAliases:
+        {{- toYaml .Values.deployment.hostAliases | nindent 6 }}
+      {{- end}}
+      {{- if .Values.dnsPolicy }}
+      dnsPolicy: {{ .Values.dnsPolicy | quote }}
+      {{- end }}
+      {{- if .Values.dnsConfig }}
+      dnsConfig:
+{{ toYaml .Values.dnsConfig | indent 8 }}
+      {{- end }}
+      containers:
+      {{- if .Values.ingressController.enabled }}
+      {{- include "kong.controller-container" . | nindent 6 }}
+      {{ end }}
+      {{- if .Values.deployment.sidecarContainers }}
+      {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
+      {{- end }}
+      {{- if .Values.deployment.kong.enabled }}
+      - name: "proxy"
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        env:
+        {{- include "kong.no_daemon_env" . | nindent 8 }}
+        lifecycle:
+          {{- toYaml .Values.lifecycle | nindent 10 }}
+        ports:
+        {{- if (and .Values.admin.http.enabled .Values.admin.enabled) }}
+        - name: admin
+          containerPort: {{ .Values.admin.http.containerPort }}
+          {{- if .Values.admin.http.hostPort }}
+          hostPort: {{ .Values.admin.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.admin.tls.enabled .Values.admin.enabled) }}
+        - name: admin-tls
+          containerPort: {{ .Values.admin.tls.containerPort }}
+          {{- if .Values.admin.tls.hostPort }}
+          hostPort: {{ .Values.admin.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.http.enabled .Values.proxy.enabled) }}
+        - name: proxy
+          containerPort: {{ .Values.proxy.http.containerPort }}
+          {{- if .Values.proxy.http.hostPort }}
+          hostPort: {{ .Values.proxy.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.tls.enabled .Values.proxy.enabled)}}
+        - name: proxy-tls
+          containerPort: {{ .Values.proxy.tls.containerPort }}
+          {{- if .Values.proxy.tls.hostPort }}
+          hostPort: {{ .Values.proxy.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- range .Values.proxy.stream }}
+        - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- range .Values.udpProxy.stream }}
+        - name: streamudp-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- if (and .Values.status.http.enabled .Values.status.enabled)}}
+        - name: status
+          containerPort: {{ .Values.status.http.containerPort }}
+          {{- if .Values.status.http.hostPort }}
+          hostPort: {{ .Values.status.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.status.tls.enabled .Values.status.enabled) }}
+        - name: status-tls
+          containerPort: {{ .Values.status.tls.containerPort }}
+          {{- if .Values.status.tls.hostPort }}
+          hostPort: {{ .Values.status.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.cluster.tls.enabled .Values.cluster.enabled) }}
+        - name: cluster-tls
+          containerPort: {{ .Values.cluster.tls.containerPort }}
+          {{- if .Values.cluster.tls.hostPort }}
+          hostPort: {{ .Values.cluster.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if .Values.enterprise.enabled }}
+        {{- if (and .Values.manager.http.enabled .Values.manager.enabled) }}
+        - name: manager
+          containerPort: {{ .Values.manager.http.containerPort }}
+          {{- if .Values.manager.http.hostPort }}
+          hostPort: {{ .Values.manager.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.manager.tls.enabled .Values.manager.enabled) }}
+        - name: manager-tls
+          containerPort: {{ .Values.manager.tls.containerPort }}
+          {{- if .Values.manager.tls.hostPort }}
+          hostPort: {{ .Values.manager.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.http.enabled .Values.portal.enabled) }}
+        - name: portal
+          containerPort: {{ .Values.portal.http.containerPort }}
+          {{- if .Values.portal.http.hostPort }}
+          hostPort: {{ .Values.portal.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.tls.enabled .Values.portal.enabled) }}
+        - name: portal-tls
+          containerPort: {{ .Values.portal.tls.containerPort }}
+          {{- if .Values.portal.tls.hostPort }}
+          hostPort: {{ .Values.portal.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.http.enabled .Values.portalapi.enabled) }}
+        - name: portalapi
+          containerPort: {{ .Values.portalapi.http.containerPort }}
+          {{- if .Values.portalapi.http.hostPort }}
+          hostPort: {{ .Values.portalapi.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.tls.enabled .Values.portalapi.enabled) }}
+        - name: portalapi-tls
+          containerPort: {{ .Values.portalapi.tls.containerPort }}
+          {{- if .Values.portalapi.tls.hostPort }}
+          hostPort: {{ .Values.portalapi.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.clustertelemetry.tls.enabled .Values.clustertelemetry.enabled) }}
+        - name: clustert-tls
+          containerPort: {{ .Values.clustertelemetry.tls.containerPort }}
+          {{- if .Values.clustertelemetry.tls.hostPort }}
+          hostPort: {{ .Values.clustertelemetry.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- end }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 10 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 10 }}
+        readinessProbe:
+{{ include "kong.proxy.compatibleReadiness" . | indent 10 }}
+        livenessProbe:
+{{ toYaml .Values.livenessProbe | indent 10 }}
+        {{- if .Values.startupProbe }}
+        startupProbe:
+{{ toYaml .Values.startupProbe | indent 10 }}
+        {{- end }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        {{- end }} {{/* End of Kong container spec */}}
+    {{- if .Values.affinity }}
+      affinity:
+{{ toYaml .Values.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+{{ toYaml .Values.topologySpreadConstraints | indent 8 }}
+    {{- end }}
+      securityContext:
+      {{- include "kong.podsecuritycontext" . | nindent 8 }}
+    {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+    {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+    {{- if .Values.tolerations }}
+      tolerations:
+{{ toYaml .Values.tolerations | indent 8 }}
+    {{- end }}
+      volumes:
+      {{- include "kong.volumes" . | nindent 8 -}}
+      {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
+{{- end }}
+
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 7d07accdf..c7e8ce267 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -125,6 +125,7 @@ deployment:
   #  name:
   #  annotations: {}
 
+  # TODO 921 pod section
   ## Optionally specify any extra sidecar containers to be included in the deployment
   ## See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core
   # sidecarContainers:
@@ -139,6 +140,7 @@ deployment:
   #   - "foo.local"
   #   - "bar.local"
 
+  # TODO 921 pod section
   ## Define any volumes and mounts you want present in the Kong proxy container
   # userDefinedVolumes:
   # - name: "volumeName"
@@ -1118,7 +1120,7 @@ podSecurityPolicy:
     # If you use Kong Enterprise <1.5, this must be set to false.
     readOnlyRootFilesystem: true
 
-# TODO 921 figure out what this is and where it should go
+# TODO 921 .deployment.*.pod.priorityClassName
 priorityClassName: ""
 
 # TODO 921 deployments.*.pod.securityContext

From 762980eeecec3dc5287135741fcd8552c3db823c Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:35:39 -0800
Subject: [PATCH 03/24] wip: start adding function call deployment

---
 charts/kong/templates/controller-deploy.yaml | 23 ++++++++++++++++++++
 charts/kong/values.yaml                      |  1 +
 2 files changed, 24 insertions(+)
 create mode 100644 charts/kong/templates/controller-deploy.yaml

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
new file mode 100644
index 000000000..9ee2973d8
--- /dev/null
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.deployment.kong.enabled }}
+{{- $deployConfig := dict -}}
+{{- $podConfig := dict -}}
+{{- $_ := set $deployConfig "daemonset" .deployment.controller.daemonset -}}
+{{- $_ := set $deployConfig "name" (printf "%s-%s (include "kong.fullname" .) "controller") -}}
+{{- $_ := set $deployConfig "namespace" (include "kong.namespace" .) -}}
+# TODO 921 combine with .deployment.controller.labels.
+# unsure how the types work out here--kong.metaLabels produces YAML output, and I'm not sure if we can
+# just concat them as strings or what. can maybe ignore .deployment.controller.labels--it wasn't
+# included in the past and we instead set extraLabels on EVERYTHING. that's apparently sufficient for
+# user needs (there's been no request to provide separate per-resource labels here), but incongruous
+# with how we handle annotations
+{{- $_ := set $deployConfig "labels" (include "kong.metaLabels" .) -}}
+{{- $_ := set $deployConfig "annotations" .deployment.controller.annotations -}}
+{{- $_ := set $deployConfig "autoscaling" .deployment.controller.autoscaling -}}
+{{- $_ := set $deployConfig "replicaCount" .deployment.controller.replicaCount -}}
+# TODO 921 this needs changes to the helper template since we'll need to point to two different sets of pods.
+# in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
+# need to pass a suffix to the helper
+{{- $_ := set $deployConfig "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
+{{- $_ := set $deployConfig "updateStrategy" .deployment.controller.updateStrategy -}}
+{{- $_ := set $deployConfig "minReadySeconds" .deployment.controller.updateStrategy -}}
+
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index c7e8ce267..1a22684b5 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -56,6 +56,7 @@
 # controller Deployment, but `.podAnnotations` for the Kong Deployment's Pods.
 deployment:
   controller:
+    labels: {}
     annotations: {} # TODO 921 add any defaults
     # TODO 921 the equivalent root setting has "default" values: https://github.com/Kong/charts/blob/kong-2.30.0/charts/kong/values.yaml#L887-L904
     # These are no-oped with an `enabled=false` key, but are somewhat

From 65ceea1586d6df579b49c2626af6687a9cf27753 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:36:44 -0800
Subject: [PATCH 04/24] wip: merge unmodified keys for deployment call

---
 charts/kong/templates/controller-deploy.yaml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index 9ee2973d8..b5ce5c021 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -1,7 +1,7 @@
 {{- if .Values.deployment.kong.enabled }}
 {{- $deployConfig := dict -}}
+{{- $deployConfig = (mustMerge $deployConfig .deployment.controller) -}}
 {{- $podConfig := dict -}}
-{{- $_ := set $deployConfig "daemonset" .deployment.controller.daemonset -}}
 {{- $_ := set $deployConfig "name" (printf "%s-%s (include "kong.fullname" .) "controller") -}}
 {{- $_ := set $deployConfig "namespace" (include "kong.namespace" .) -}}
 # TODO 921 combine with .deployment.controller.labels.
@@ -11,13 +11,8 @@
 # user needs (there's been no request to provide separate per-resource labels here), but incongruous
 # with how we handle annotations
 {{- $_ := set $deployConfig "labels" (include "kong.metaLabels" .) -}}
-{{- $_ := set $deployConfig "annotations" .deployment.controller.annotations -}}
-{{- $_ := set $deployConfig "autoscaling" .deployment.controller.autoscaling -}}
-{{- $_ := set $deployConfig "replicaCount" .deployment.controller.replicaCount -}}
 # TODO 921 this needs changes to the helper template since we'll need to point to two different sets of pods.
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
 {{- $_ := set $deployConfig "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
-{{- $_ := set $deployConfig "updateStrategy" .deployment.controller.updateStrategy -}}
-{{- $_ := set $deployConfig "minReadySeconds" .deployment.controller.updateStrategy -}}
 

From b4654e9bdcff6e78c0d49f7510307053fe785ea6 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:42:47 -0800
Subject: [PATCH 05/24] wip: SAT name

---
 charts/kong/templates/controller-deploy.yaml     |  2 +-
 charts/kong/templates/deployment-controller.yaml | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index b5ce5c021..f3414fa27 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -15,4 +15,4 @@
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
 {{- $_ := set $deployConfig "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
-
+{{- $_ := set $deployConfig "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index b81ed54ec..cc81a2f22 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -6,7 +6,7 @@ kind: DaemonSet
 kind: Deployment
 {{- end }}
 metadata:
-  name: {{ template "kong.fullname" . }} # TODO 921 needs suffix for deployment via arg. also needs to be passed in
+  name: {{ .name }} # TODO 921 needs suffix for deployment via arg. also needs to be passed in
   namespace:  {{ template "kong.namespace" . }}
   labels:
     {{- include "kong.metaLabels" . | nindent 4 }} # TODO 921 needs to be passed in for root
@@ -32,7 +32,7 @@ spec:
   {{- else }}
   strategy:
   {{- end }}
-{{ toYaml .Values.updateStrategy | indent 4 }}
+{{ toYaml .updateStrategy | indent 4 }}
   {{- end }}
   {{- if .minReadySeconds }}
   minReadySeconds: {{ .minReadySeconds }}
@@ -45,8 +45,10 @@ spec:
              # there's no obvious reason to separate them, since there can only be one per Deployment anyway */}}
         {{- if (and (not .serviceAccount.automountServiceAccountToken) (or .serviceAccount.create .serviceAccount.name)) }}
         {{/* # TODO 921 needs to be passed in as-is. possibly should be made relative, since the generator will attempt to use the same
-             # name, but lives in a separate service-account.yaml template, so vov */}}
-        kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
+             # name, but lives in a separate service-account.yaml template, so vov
+             # TODO above is complete but keeping the note around for now
+        */}}
+        kuma.io/service-account-token-volume: {{ .serviceAccountTokenName }}
         {{- end }}
         {{/*  # TODO 921 special to DB-less Kong with static config. I think this can be handled with the .Values.dblessConfig.config only:
               # if you do not want to redeploy on hash change, do not provide a static config. However, this would fire in both controller

From 9eddd6b12fb78ebd605dde8065e8dba7440d57cb Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 13:53:13 -0800
Subject: [PATCH 06/24] wip: separate merge and rendered keys

---
 charts/kong/templates/controller-deploy.yaml     | 13 ++++++++-----
 charts/kong/templates/deployment-controller.yaml | 10 +++++-----
 2 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index f3414fa27..a8a58b6db 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -1,18 +1,21 @@
 {{- if .Values.deployment.kong.enabled }}
 {{- $deployConfig := dict -}}
 {{- $deployConfig = (mustMerge $deployConfig .deployment.controller) -}}
+{{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
 {{- $podConfig := dict -}}
-{{- $_ := set $deployConfig "name" (printf "%s-%s (include "kong.fullname" .) "controller") -}}
-{{- $_ := set $deployConfig "namespace" (include "kong.namespace" .) -}}
+{{- $_ := set $deployPreRender "name" (printf "%s-%s (include "kong.fullname" .) "controller") -}}
+{{- $_ := set $deployPreRender "namespace" (include "kong.namespace" .) -}}
 # TODO 921 combine with .deployment.controller.labels.
 # unsure how the types work out here--kong.metaLabels produces YAML output, and I'm not sure if we can
 # just concat them as strings or what. can maybe ignore .deployment.controller.labels--it wasn't
 # included in the past and we instead set extraLabels on EVERYTHING. that's apparently sufficient for
 # user needs (there's been no request to provide separate per-resource labels here), but incongruous
 # with how we handle annotations
-{{- $_ := set $deployConfig "labels" (include "kong.metaLabels" .) -}}
+{{- $_ := set $deployPreRender "labels" (include "kong.metaLabels" .) -}}
 # TODO 921 this needs changes to the helper template since we'll need to point to two different sets of pods.
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
-{{- $_ := set $deployConfig "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
-{{- $_ := set $deployConfig "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
+{{- $_ := set $deployPreRender "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
+
+{{- $_ := set $deployConfig "pre" $deployPreRender -}} # TODO 921 maybe think of a better key name
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index cc81a2f22..ce4e677b0 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -6,10 +6,10 @@ kind: DaemonSet
 kind: Deployment
 {{- end }}
 metadata:
-  name: {{ .name }} # TODO 921 needs suffix for deployment via arg. also needs to be passed in
-  namespace:  {{ template "kong.namespace" . }}
+  name: {{ .pre.name }}
+  namespace:  {{ .pre.namespace }}
   labels:
-    {{- include "kong.metaLabels" . | nindent 4 }} # TODO 921 needs to be passed in for root
+    {{- .pre.labels | nindent 4 }}
     app.kubernetes.io/component: app
   {{- if .annotations }}
   annotations:
@@ -25,7 +25,7 @@ spec:
   {{- end }}
   selector:
     matchLabels:
-      {{- include "kong.selectorLabels" . | nindent 6 }} # TODO 921 needs to be passed in
+      {{- .pre.selectorMatchLabels }}
   {{- if .updateStrategy }}
   {{- if .daemonset }}
   updateStrategy:
@@ -48,7 +48,7 @@ spec:
              # name, but lives in a separate service-account.yaml template, so vov
              # TODO above is complete but keeping the note around for now
         */}}
-        kuma.io/service-account-token-volume: {{ .serviceAccountTokenName }}
+        kuma.io/service-account-token-volume: {{ .pre.serviceAccountTokenName }}
         {{- end }}
         {{/*  # TODO 921 special to DB-less Kong with static config. I think this can be handled with the .Values.dblessConfig.config only:
               # if you do not want to redeploy on hash change, do not provide a static config. However, this would fire in both controller

From 31a1838d0e9f7265dfeecc3680f3eee0d76cefd9 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:13:50 -0800
Subject: [PATCH 07/24] wip: separate deploy and pod prerender keys

---
 charts/kong/templates/controller-deploy.yaml  | 10 +++++++---
 .../kong/templates/deployment-controller.yaml | 20 +++++++++----------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index a8a58b6db..7ea1b5e52 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -1,8 +1,9 @@
-{{- if .Values.deployment.kong.enabled }}
+{{- if .Values.deployment.controller.enabled }}
 {{- $deployConfig := dict -}}
 {{- $deployConfig = (mustMerge $deployConfig .deployment.controller) -}}
 {{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
-{{- $podConfig := dict -}}
+{{- $podPreRender := dict -}}
+
 {{- $_ := set $deployPreRender "name" (printf "%s-%s (include "kong.fullname" .) "controller") -}}
 {{- $_ := set $deployPreRender "namespace" (include "kong.namespace" .) -}}
 # TODO 921 combine with .deployment.controller.labels.
@@ -18,4 +19,7 @@
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
 
-{{- $_ := set $deployConfig "pre" $deployPreRender -}} # TODO 921 maybe think of a better key name
+{{- $preRender := dict -}}
+{{- $_ := set $preRender "deployment" $deployPreRender -}}
+{{- $_ := set $preRender "pod" $podPreRender -}}
+{{- $_ := set $deployConfig "pre" $preRender -}} # TODO 921 maybe think of a better key name
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index ce4e677b0..8dbe29d3e 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -6,10 +6,10 @@ kind: DaemonSet
 kind: Deployment
 {{- end }}
 metadata:
-  name: {{ .pre.name }}
-  namespace:  {{ .pre.namespace }}
+  name: {{ .pre.deployment.name }}
+  namespace:  {{ .pre.deployment.namespace }}
   labels:
-    {{- .pre.labels | nindent 4 }}
+    {{- .pre.deployment.labels | nindent 4 }}
     app.kubernetes.io/component: app
   {{- if .annotations }}
   annotations:
@@ -25,7 +25,7 @@ spec:
   {{- end }}
   selector:
     matchLabels:
-      {{- .pre.selectorMatchLabels }}
+      {{- .pre.deployment.selectorMatchLabels }}
   {{- if .updateStrategy }}
   {{- if .daemonset }}
   updateStrategy:
@@ -42,24 +42,22 @@ spec:
     metadata:
       annotations:
         {{/* # TODO 921 some keys in the pod template end up being from the deployment dict, so probably no separate define 
-             # there's no obvious reason to separate them, since there can only be one per Deployment anyway */}}
+             # there's no obvious reason to separate them, since there can only be one per Deployment anyway
+        */}}
         {{- if (and (not .serviceAccount.automountServiceAccountToken) (or .serviceAccount.create .serviceAccount.name)) }}
         {{/* # TODO 921 needs to be passed in as-is. possibly should be made relative, since the generator will attempt to use the same
              # name, but lives in a separate service-account.yaml template, so vov
              # TODO above is complete but keeping the note around for now
         */}}
-        kuma.io/service-account-token-volume: {{ .pre.serviceAccountTokenName }}
+        kuma.io/service-account-token-volume: {{ .pre.deployment.serviceAccountTokenName }}
         {{- end }}
         {{/*  # TODO 921 special to DB-less Kong with static config. I think this can be handled with the .Values.dblessConfig.config only:
               # if you do not want to redeploy on hash change, do not provide a static config. However, this would fire in both controller
               # and proxy calls, so we probably need to have injection for additional Pod (and whatever else) annotations.
         */}}
-        {{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off" )) }}
-        {{- if .Values.dblessConfig.config }}
-        checksum/dbless.config: {{ toYaml .Values.dblessConfig.config | sha256sum }}
-        {{- end }}
+        {{- if .pre.deployment.dblessChecksum }}
+        checksum/dbless.config: {{ .pre.deployment.dblessChecksum }}
         {{- end }}
-        {{/* # TODO 921 end weird config hash block */}}
         {{- if .pod.annotations }}
         {{- range $key, $value := .pod.annotations }}
         {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}

From 5dcbce69d7dc3e2a8fd3ae2d7cd085264d3f4d2a Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:25:53 -0800
Subject: [PATCH 08/24] wip: some pod stuff

---
 charts/kong/templates/controller-deploy.yaml     |  5 +++++
 charts/kong/templates/deployment-controller.yaml | 10 +++++-----
 charts/kong/values.yaml                          |  2 ++
 3 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index 7ea1b5e52..d1fbf09d6 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -19,6 +19,11 @@
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
 
+# TODO 921 ditto the deploy labels todo
+{{- $_ := set $podPreRender "labels" (include "kong.metaLabels" .) -}}
+{{- $_ := set $podPreRender "name" (printf "%s-%s (include "kong.fullname" .) "controller" -}}
+{{- $_ := set $podPreRender "versionLabel" .Chart.AppVersion -}}
+
 {{- $preRender := dict -}}
 {{- $_ := set $preRender "deployment" $deployPreRender -}}
 {{- $_ := set $preRender "pod" $podPreRender -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 8dbe29d3e..f47fab909 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -64,22 +64,22 @@ spec:
         {{- end }}
         {{- end }}
       labels:
-        {{- include "kong.metaLabels" . | nindent 8 }} # TODO 921 passed in
+        {{- .pre.pod.labels | nindent 8 }}
         app.kubernetes.io/component: app
-        app: {{ template "kong.fullname" . }} # TODO 921 passed in, needs suffix
-        version: {{ .Chart.AppVersion | quote }} # TODO 921 passed in
+        app: {{ .pre.pod.name }}
+        version: {{ .pre.pod.versionLabel | quote }}
         {{- if .pod.labels }}
         {{ include "kong.renderTpl" (dict "value" .pod.labels "context" $) | nindent 8 }}
         {{- end }}
     spec:
-      {{- if .hostNetwork }}
+      {{- if .pod.hostNetwork }}
       hostNetwork: true
       {{- end }}
       {{- if .pod.priorityClassName }}
       priorityClassName: "{{ .pod.priorityClassName }}"
       {{- end }}
       {{- if or .serviceAccount.create .serviceAccount.name }}
-      serviceAccountName: {{ template "kong.serviceAccountName" . }} # TODO 921 passed in
+      serviceAccountName: {{ .pre.deployment.serviceAccountTokenName }}
       {{- end }}
       {{- if (and (or .serviceAccount.create .serviceAccount.name) .serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 1a22684b5..4ea3cf262 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -79,7 +79,9 @@ deployment:
         # arbitrary plugin codeTODO 921 we arguably don't need this with a
         # split controller Deployment, since we use it to hide the token from
         # Kong and its arbitrary plugin code.
+        # TODO 921 probably ignore the above, I think we still need it for the Kuma case
         automountServiceAccountToken: false
+      hostNetwork: false
       container:
         env:
           # The controller disables TLS verification by default because Kong

From 605787d4219607646cb536813f8d2645dbf1940f Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:37:54 -0800
Subject: [PATCH 09/24] wip: move some top-level keys

---
 .../kong/templates/deployment-controller.yaml |  4 +-
 charts/kong/values.yaml                       | 45 ++++++++-----------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index f47fab909..7b251ae7e 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -304,10 +304,10 @@ spec:
         {{- end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
-        {{- end }} {{/* End of Kong container spec */}}
+        {{- end }} {{/* TODO 921 End of Kong container spec */}}
     {{- if .Values.affinity }}
       affinity:
-{{ toYaml .Values.affinity | indent 8 }}
+{{ toYaml .pod.affinity | indent 8 }}
     {{- end }}
     {{- if .Values.topologySpreadConstraints }}
       topologySpreadConstraints:
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 4ea3cf262..756a06205 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -70,6 +70,11 @@ deployment:
     # controller. that said, we may be able to use the same templates for both
     # Deployments' basic Deployment-y things and get it for free.
     daemonset: false
+    updateStrategy: {}
+      # type: RollingUpdate
+      # rollingUpdate:
+      #   maxSurge: "100%"
+      #   maxUnavailable: "0%"
     pod:
       annotations: {} # .podAnnotations, for the controller
       serviceAccount:
@@ -82,6 +87,18 @@ deployment:
         # TODO 921 probably ignore the above, I think we still need it for the Kuma case
         automountServiceAccountToken: false
       hostNetwork: false
+      # Affinity for pod assignment
+      # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+      affinity: {}
+      # Topology spread constraints for pod assignment (requires Kubernetes >= 1.19)
+      # Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
+      topologySpreadConstraints: []
+      # Sets the termination grace period for pods spawned by the Kubernetes Deployment.
+      # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution
+      terminationGracePeriodSeconds: 30
+      # Tolerations for pod assignment
+      # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+      tolerations: []
       container:
         env:
           # The controller disables TLS verification by default because Kong
@@ -945,13 +962,6 @@ waitImage:
   # tag: 5
   pullPolicy: IfNotPresent
 
-# TODO 921 deployment.*.updateStrategy
-# update strategy
-updateStrategy: {}
-  # type: RollingUpdate
-  # rollingUpdate:
-  #   maxSurge: "100%"
-  #   maxUnavailable: "0%"
 
 # TODO 921 deployments.kong.pod.container.resources
 # If you want to specify resources, uncomment the following
@@ -1016,25 +1026,8 @@ lifecycle:
         - quit
         - '--wait=15'
 
-# TODO 921 deployments.kong.pod.terminationGracePeriodSeconds
-# Sets the termination grace period for pods spawned by the Kubernetes Deployment.
-# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution
-terminationGracePeriodSeconds: 30
-
-# TODO 921 deployments.kong.pod.affinity
-# Affinity for pod assignment
-# Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
-# affinity: {}
-
-# TODO 921 deployments.kong.pod.topologySpreadConstraints
-# Topology spread constraints for pod assignment (requires Kubernetes >= 1.19)
-# Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
-# topologySpreadConstraints: []
-
-# TODO 921 deployments.kong.pod.tolerations
-# Tolerations for pod assignment
-# Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
-tolerations: []
+
+
 
 # TODO 921 either part of the pod or deployment (or under affinity?), not sure
 # need to look up

From 5d257c097e41d1953bbedd6c25fcb9426f378234 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 14:57:07 -0800
Subject: [PATCH 10/24] wip: move more root things to their new homes

---
 .../kong/templates/deployment-controller.yaml | 17 +++++----
 charts/kong/values.yaml                       | 37 ++++++++++---------
 2 files changed, 29 insertions(+), 25 deletions(-)

diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 7b251ae7e..16e244744 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -150,6 +150,7 @@ spec:
       {{- if .Values.deployment.sidecarContainers }}
       {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
       {{- end }}
+      # TODO 921 this presumably goes away
       {{- if .Values.deployment.kong.enabled }}
       - name: "proxy"
         image: {{ include "kong.getRepoTag" .Values.image }}
@@ -309,22 +310,24 @@ spec:
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}
     {{- end }}
-    {{- if .Values.topologySpreadConstraints }}
+    {{- if .pod.topologySpreadConstraints }}
       topologySpreadConstraints:
-{{ toYaml .Values.topologySpreadConstraints | indent 8 }}
+{{ toYaml .pod.topologySpreadConstraints | indent 8 }}
     {{- end }}
       securityContext:
+      # TODO 921 pass in
       {{- include "kong.podsecuritycontext" . | nindent 8 }}
-    {{- if .Values.nodeSelector }}
+    {{- if .pod.nodeSelector }}
       nodeSelector:
-{{ toYaml .Values.nodeSelector | indent 8 }}
+{{ toYaml .pod.nodeSelector | indent 8 }}
     {{- end }}
-      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
-    {{- if .Values.tolerations }}
+      terminationGracePeriodSeconds: {{ .pod.terminationGracePeriodSeconds }}
+    {{- if .pod.tolerations }}
       tolerations:
-{{ toYaml .Values.tolerations | indent 8 }}
+{{ toYaml .pod.tolerations | indent 8 }}
     {{- end }}
       volumes:
+      # TODO 921 pass in
       {{- include "kong.volumes" . | nindent 8 -}}
       {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
 
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 756a06205..ee7f0a764 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -75,6 +75,9 @@ deployment:
       # rollingUpdate:
       #   maxSurge: "100%"
       #   maxUnavailable: "0%"
+    sidecarContainers: []
+    userDefinedVolumes: []
+    initContainers: []
     pod:
       annotations: {} # .podAnnotations, for the controller
       serviceAccount:
@@ -99,6 +102,9 @@ deployment:
       # Tolerations for pod assignment
       # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
       tolerations: []
+      # Node labels for pod assignment
+      # Ref: https://kubernetes.io/docs/user-guide/node-selection/
+      nodeSelector: {}
       container:
         env:
           # The controller disables TLS verification by default because Kong
@@ -122,9 +128,16 @@ deployment:
           # version, set this to "3.1.0".
           effectiveSemver:
         userDefinedVolumeMounts: []
-      sidecarContainers: []
-      userDefinedVolumes: []
-      initContainers: []
+        # If you want to specify resources, uncomment the following
+        # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+        resources: {}
+          # limits:
+          #  cpu: 1
+          #  memory: 2G
+          # requests:
+          #  cpu: 1
+          #  memory: 2G
+
   kong:
     # Enable or disable Kong itself
     # Setting this to false with ingressController.enabled=true will create a
@@ -963,17 +976,10 @@ waitImage:
   pullPolicy: IfNotPresent
 
 
-# TODO 921 deployments.kong.pod.container.resources
-# If you want to specify resources, uncomment the following
-# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-resources: {}
-  # limits:
-  #  cpu: 1
-  #  memory: 2G
-  # requests:
-  #  cpu: 1
-  #  memory: 2G
 
+# TODO 921 not moving these til we have a kong new-style section
+# they need different values for the controller and I want to preserve the existing defaults
+# for now
 # TODO 921 deployments.kong.pod.container.readinessProbe
 # readinessProbe for Kong pods
 readinessProbe:
@@ -1029,11 +1035,6 @@ lifecycle:
 
 
 
-# TODO 921 either part of the pod or deployment (or under affinity?), not sure
-# need to look up
-# Node labels for pod assignment
-# Ref: https://kubernetes.io/docs/user-guide/node-selection/
-nodeSelector: {}
 
 # TODO 921 deployments.kong.pod.annotations
 # Annotation to be added to Kong pods

From f8b4b39796520a2f374cabcfb767b63c0929de62 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:02:58 -0800
Subject: [PATCH 11/24] wip: move pod security context

---
 charts/kong/templates/_helpers.tpl               | 7 -------
 charts/kong/templates/deployment-controller.yaml | 4 ++--
 charts/kong/values.yaml                          | 4 +---
 3 files changed, 3 insertions(+), 12 deletions(-)

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index ee7777ab3..04f2b678c 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -1008,13 +1008,6 @@ valueFrom:
     key: {{ .key }}
 {{- end -}}
 
-{{/*
-Use the Pod security context defined in Values or set the UID by default
-*/}}
-{{- define "kong.podsecuritycontext" -}}
-{{ .Values.securityContext | toYaml }}
-{{- end -}}
-
 {{- define "kong.no_daemon_env" -}}
 {{- template "kong.env" . }}
 - name: KONG_NGINX_DAEMON
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 16e244744..7e62dc1e5 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -315,8 +315,8 @@ spec:
 {{ toYaml .pod.topologySpreadConstraints | indent 8 }}
     {{- end }}
       securityContext:
-      # TODO 921 pass in
-      {{- include "kong.podsecuritycontext" . | nindent 8 }}
+      # TODO 921 this previously used a helper but it only did the below and seemed kinda useless
+      {{- .pod.securityContext | toYaml | nindent 8 }}
     {{- if .pod.nodeSelector }}
       nodeSelector:
 {{ toYaml .pod.nodeSelector | indent 8 }}
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index ee7f0a764..607876ac4 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -105,6 +105,7 @@ deployment:
       # Node labels for pod assignment
       # Ref: https://kubernetes.io/docs/user-guide/node-selection/
       nodeSelector: {}
+      securityContext: {}
       container:
         env:
           # The controller disables TLS verification by default because Kong
@@ -1120,9 +1121,6 @@ podSecurityPolicy:
 # TODO 921 .deployment.*.pod.priorityClassName
 priorityClassName: ""
 
-# TODO 921 deployments.*.pod.securityContext
-# securityContext for Kong pods.
-securityContext: {}
 
 # TODO 921 deployments.*.pod.container.securityContext
 # securityContext for containers.

From 0e9eaf529e6fb5829d7f4f89f0d4267f91dbe651 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:08:44 -0800
Subject: [PATCH 12/24] wip: volumes, sorta

---
 charts/kong/templates/controller-deploy.yaml     | 5 +++++
 charts/kong/templates/deployment-controller.yaml | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index d1fbf09d6..611cab615 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -18,6 +18,11 @@
 # need to pass a suffix to the helper
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
+# TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
+# pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
+# unnecessary volumes doesn't really hurt us
+{{- $_ := set $deployPreRender "volumes" (template "kong.volumes" .) -}}
+{{- $_ := set $deployPreRender "userVolumes" (template "kong.userDefinedVolumes" .) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "labels" (include "kong.metaLabels" .) -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 7e62dc1e5..d86794ab7 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -328,8 +328,8 @@ spec:
     {{- end }}
       volumes:
       # TODO 921 pass in
-      {{- include "kong.volumes" . | nindent 8 -}}
-      {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
+      {{- .pre.pod.volumes | nindent 8 -}}
+      {{- .pre.pod.userVolumes | nindent 8 -}}
 
 {{- end -}} {{/* end kong.deployment */}}
 

From ea5cf264bde27a68f50b089a35080c46bb17ce2b Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:11:43 -0800
Subject: [PATCH 13/24] wip: disable old deployment template

---
 charts/kong/templates/deployment.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/charts/kong/templates/deployment.yaml b/charts/kong/templates/deployment.yaml
index 0aa46ceea..3ab1fb8c5 100644
--- a/charts/kong/templates/deployment.yaml
+++ b/charts/kong/templates/deployment.yaml
@@ -1,3 +1,4 @@
+{{/*
 {{- if or .Values.deployment.kong.enabled .Values.ingressController.enabled }}
 apiVersion: apps/v1
 {{- if .Values.deployment.daemonset }}
@@ -303,3 +304,4 @@ spec:
       {{- include "kong.volumes" . | nindent 8 -}}
       {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
 {{- end }}
+*/}}

From 287b9fee948d3205426fb7a3fdc39e950ae406b4 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:13:03 -0800
Subject: [PATCH 14/24] wip: temporarily disable container

---
 charts/kong/templates/deployment-controller.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index d86794ab7..832fd37fe 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -135,7 +135,8 @@ spec:
       dnsConfig:
 {{ toYaml .pod.dnsConfig | indent 8 }}
       {{- end }}
-      containers:
+      containers: {}
+      {{/* CONTAINERS CURRENTLY DISABLED
       # TODO 921 all sorts of fun stuff in this section. .pod.container
       # contains generic configuration, but in practice this needs to be
       # combined with a bunch of app-specific config that lives elsewhere, and
@@ -306,6 +307,7 @@ spec:
         resources:
 {{ toYaml .Values.resources | indent 10 }}
         {{- end }} {{/* TODO 921 End of Kong container spec */}}
+CONTAINERS CURRENTLY DISABLED */}}
     {{- if .Values.affinity }}
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}

From d4d4a240666ea92b99c33ce7d95206abbf62c6e7 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:24:36 -0800
Subject: [PATCH 15/24] wip: put wips in jail

Helm is being mean and not honoring comment tags. Why? No idea. Not
bothering to figure it out and instead just dumping WIP stuff that can't
render in a jail directory so that Helm stops complaining about
misplaced "{{- end -}}" things it shouldn't be trying to render.
---
 .../kong/{templates => jail}/deployment.yaml  |   0
 charts/kong/jail/tmp-container-block.yaml     | 172 +++++++
 charts/kong/jail/tmp-extra-deploy.yaml        | 307 +++++++++++
 charts/kong/jail/tmp-kong-deploy.yaml         |  36 ++
 charts/kong/templates/controller-deploy.yaml  |   2 +
 .../kong/templates/deployment-controller.yaml | 479 ------------------
 6 files changed, 517 insertions(+), 479 deletions(-)
 rename charts/kong/{templates => jail}/deployment.yaml (100%)
 create mode 100644 charts/kong/jail/tmp-container-block.yaml
 create mode 100644 charts/kong/jail/tmp-extra-deploy.yaml
 create mode 100644 charts/kong/jail/tmp-kong-deploy.yaml

diff --git a/charts/kong/templates/deployment.yaml b/charts/kong/jail/deployment.yaml
similarity index 100%
rename from charts/kong/templates/deployment.yaml
rename to charts/kong/jail/deployment.yaml
diff --git a/charts/kong/jail/tmp-container-block.yaml b/charts/kong/jail/tmp-container-block.yaml
new file mode 100644
index 000000000..4fc36bef7
--- /dev/null
+++ b/charts/kong/jail/tmp-container-block.yaml
@@ -0,0 +1,172 @@
+      # TODO 921 all sorts of fun stuff in this section. .pod.container
+      # contains generic configuration, but in practice this needs to be
+      # combined with a bunch of app-specific config that lives elsewhere, and
+      # not just in env. a generic function in context is probably quite
+      # difficult, though doable if you provide pass-in points for everything
+      # (mounts, env, god knows what else) and assign a var at the beginning.
+      # probably just use dedicated functions for kong and controller
+      # containers for now, with a TODO to genericize further
+      {{- if .Values.ingressController.enabled }}
+      {{- include "kong.controller-container" . | nindent 6 }}
+      {{ end }}
+      {{- if .Values.deployment.sidecarContainers }}
+      {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
+      {{- end }}
+      # TODO 921 this presumably goes away
+      {{- if .Values.deployment.kong.enabled }}
+      - name: "proxy"
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        env:
+        {{- include "kong.no_daemon_env" . | nindent 8 }}
+        lifecycle:
+          {{- toYaml .Values.lifecycle | nindent 10 }}
+        ports:
+        {{- if (and .Values.admin.http.enabled .Values.admin.enabled) }}
+        - name: admin
+          containerPort: {{ .Values.admin.http.containerPort }}
+          {{- if .Values.admin.http.hostPort }}
+          hostPort: {{ .Values.admin.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.admin.tls.enabled .Values.admin.enabled) }}
+        - name: admin-tls
+          containerPort: {{ .Values.admin.tls.containerPort }}
+          {{- if .Values.admin.tls.hostPort }}
+          hostPort: {{ .Values.admin.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.http.enabled .Values.proxy.enabled) }}
+        - name: proxy
+          containerPort: {{ .Values.proxy.http.containerPort }}
+          {{- if .Values.proxy.http.hostPort }}
+          hostPort: {{ .Values.proxy.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.tls.enabled .Values.proxy.enabled)}}
+        - name: proxy-tls
+          containerPort: {{ .Values.proxy.tls.containerPort }}
+          {{- if .Values.proxy.tls.hostPort }}
+          hostPort: {{ .Values.proxy.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- range .Values.proxy.stream }}
+        - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- range .Values.udpProxy.stream }}
+        - name: streamudp-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- if (and .Values.status.http.enabled .Values.status.enabled)}}
+        - name: status
+          containerPort: {{ .Values.status.http.containerPort }}
+          {{- if .Values.status.http.hostPort }}
+          hostPort: {{ .Values.status.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.status.tls.enabled .Values.status.enabled) }}
+        - name: status-tls
+          containerPort: {{ .Values.status.tls.containerPort }}
+          {{- if .Values.status.tls.hostPort }}
+          hostPort: {{ .Values.status.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.cluster.tls.enabled .Values.cluster.enabled) }}
+        - name: cluster-tls
+          containerPort: {{ .Values.cluster.tls.containerPort }}
+          {{- if .Values.cluster.tls.hostPort }}
+          hostPort: {{ .Values.cluster.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if .Values.enterprise.enabled }}
+        {{- if (and .Values.manager.http.enabled .Values.manager.enabled) }}
+        - name: manager
+          containerPort: {{ .Values.manager.http.containerPort }}
+          {{- if .Values.manager.http.hostPort }}
+          hostPort: {{ .Values.manager.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.manager.tls.enabled .Values.manager.enabled) }}
+        - name: manager-tls
+          containerPort: {{ .Values.manager.tls.containerPort }}
+          {{- if .Values.manager.tls.hostPort }}
+          hostPort: {{ .Values.manager.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.http.enabled .Values.portal.enabled) }}
+        - name: portal
+          containerPort: {{ .Values.portal.http.containerPort }}
+          {{- if .Values.portal.http.hostPort }}
+          hostPort: {{ .Values.portal.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.tls.enabled .Values.portal.enabled) }}
+        - name: portal-tls
+          containerPort: {{ .Values.portal.tls.containerPort }}
+          {{- if .Values.portal.tls.hostPort }}
+          hostPort: {{ .Values.portal.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.http.enabled .Values.portalapi.enabled) }}
+        - name: portalapi
+          containerPort: {{ .Values.portalapi.http.containerPort }}
+          {{- if .Values.portalapi.http.hostPort }}
+          hostPort: {{ .Values.portalapi.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.tls.enabled .Values.portalapi.enabled) }}
+        - name: portalapi-tls
+          containerPort: {{ .Values.portalapi.tls.containerPort }}
+          {{- if .Values.portalapi.tls.hostPort }}
+          hostPort: {{ .Values.portalapi.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.clustertelemetry.tls.enabled .Values.clustertelemetry.enabled) }}
+        - name: clustert-tls
+          containerPort: {{ .Values.clustertelemetry.tls.containerPort }}
+          {{- if .Values.clustertelemetry.tls.hostPort }}
+          hostPort: {{ .Values.clustertelemetry.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- end }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 10 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 10 }}
+        readinessProbe:
+{{ include "kong.proxy.compatibleReadiness" . | indent 10 }}
+        livenessProbe:
+{{ toYaml .Values.livenessProbe | indent 10 }}
+        {{- if .Values.startupProbe }}
+        startupProbe:
+{{ toYaml .Values.startupProbe | indent 10 }}
+        {{- end }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        {{- end }} {{/* TODO 921 End of Kong container spec */}}
+CONTAINERS CURRENTLY DISABLED */}}
+
diff --git a/charts/kong/jail/tmp-extra-deploy.yaml b/charts/kong/jail/tmp-extra-deploy.yaml
new file mode 100644
index 000000000..f35c1a385
--- /dev/null
+++ b/charts/kong/jail/tmp-extra-deploy.yaml
@@ -0,0 +1,307 @@
+{{/* TODO 921 replace this with a template invocation
+{{- if .Values.deployment.controller.enabled }}
+apiVersion: apps/v1
+{{- if .Values.deployment.daemonset }}
+kind: DaemonSet
+{{- else }}
+kind: Deployment
+{{- end }}
+metadata:
+  name: {{ template "kong.fullname" . }}
+  namespace:  {{ template "kong.namespace" . }}
+  labels:
+    {{- include "kong.metaLabels" . | nindent 4 }}
+    app.kubernetes.io/component: app
+  {{- if .Values.deploymentAnnotations }}
+  annotations:
+  {{- range $key, $value := .Values.deploymentAnnotations }}
+    {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
+  {{- end }}
+  {{- end }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  {{- if not .Values.deployment.daemonset }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "kong.selectorLabels" . | nindent 6 }}
+  {{- if .Values.updateStrategy }}
+  {{- if .Values.deployment.daemonset }}
+  updateStrategy:
+  {{- else }}
+  strategy:
+  {{- end }}
+{{ toYaml .Values.updateStrategy | indent 4 }}
+  {{- end }}
+  {{- if .Values.deployment.minReadySeconds }}
+  minReadySeconds: {{ .Values.deployment.minReadySeconds }}
+  {{- end }}
+
+  template:
+    metadata:
+      annotations:
+        {{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+        kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
+        {{- end }}
+        {{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off" )) }}
+        {{- if .Values.dblessConfig.config }}
+        checksum/dbless.config: {{ toYaml .Values.dblessConfig.config | sha256sum }}
+        {{- end }}
+        {{- end }}
+        {{- if .Values.podAnnotations }}
+        {{- range $key, $value := .Values.podAnnotations }}
+        {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
+        {{- end }}
+        {{- end }}
+      labels:
+        {{- include "kong.metaLabels" . | nindent 8 }}
+        app.kubernetes.io/component: app
+        app: {{ template "kong.fullname" . }}
+        version: {{ .Chart.AppVersion | quote }}
+        {{- if .Values.podLabels }}
+        {{ include "kong.renderTpl" (dict "value" .Values.podLabels "context" $) | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- if .Values.deployment.hostNetwork }}
+      hostNetwork: true
+      {{- end }}
+      {{- if .Values.priorityClassName }}
+      priorityClassName: "{{ .Values.priorityClassName }}"
+      {{- end }}
+      {{- if or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name }}
+      serviceAccountName: {{ template "kong.serviceAccountName" . }}
+      {{- end }}
+      {{- if (and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) .Values.deployment.serviceAccount.automountServiceAccountToken) }}
+      automountServiceAccountToken: true
+      {{- else }}
+      automountServiceAccountToken: false
+      {{ end }}
+      {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+      {{- range .Values.image.pullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      {{- if .Values.deployment.kong.enabled }}
+      initContainers:
+      - name: clear-stale-pid
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        command:
+        - "rm"
+        - "-vrf"
+        - "$KONG_PREFIX/pids"
+        env:
+        {{- include "kong.env" . | nindent 8 }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 8 }}
+        {{- if .Values.deployment.initContainers }}
+        {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+        {{- end }}
+        {{- if (and (not (eq .Values.env.database "off")) .Values.waitImage.enabled) }}
+        {{- include "kong.wait-for-db" . | nindent 6 }}
+        {{- end }}
+      {{- end }}
+      {{- if .Values.deployment.hostAliases }}
+      hostAliases:
+        {{- toYaml .Values.deployment.hostAliases | nindent 6 }}
+      {{- end}}
+      {{- if .Values.dnsPolicy }}
+      dnsPolicy: {{ .Values.dnsPolicy | quote }}
+      {{- end }}
+      {{- if .Values.dnsConfig }}
+      dnsConfig:
+{{ toYaml .Values.dnsConfig | indent 8 }}
+      {{- end }}
+      containers:
+      {{- if .Values.ingressController.enabled }}
+      {{- include "kong.controller-container" . | nindent 6 }}
+      {{ end }}
+      {{- if .Values.deployment.sidecarContainers }}
+      {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
+      {{- end }}
+      {{- if .Values.deployment.kong.enabled }}
+      - name: "proxy"
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        env:
+        {{- include "kong.no_daemon_env" . | nindent 8 }}
+        lifecycle:
+          {{- toYaml .Values.lifecycle | nindent 10 }}
+        ports:
+        {{- if (and .Values.admin.http.enabled .Values.admin.enabled) }}
+        - name: admin
+          containerPort: {{ .Values.admin.http.containerPort }}
+          {{- if .Values.admin.http.hostPort }}
+          hostPort: {{ .Values.admin.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.admin.tls.enabled .Values.admin.enabled) }}
+        - name: admin-tls
+          containerPort: {{ .Values.admin.tls.containerPort }}
+          {{- if .Values.admin.tls.hostPort }}
+          hostPort: {{ .Values.admin.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.http.enabled .Values.proxy.enabled) }}
+        - name: proxy
+          containerPort: {{ .Values.proxy.http.containerPort }}
+          {{- if .Values.proxy.http.hostPort }}
+          hostPort: {{ .Values.proxy.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.proxy.tls.enabled .Values.proxy.enabled)}}
+        - name: proxy-tls
+          containerPort: {{ .Values.proxy.tls.containerPort }}
+          {{- if .Values.proxy.tls.hostPort }}
+          hostPort: {{ .Values.proxy.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- range .Values.proxy.stream }}
+        - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- range .Values.udpProxy.stream }}
+        - name: streamudp-{{ .containerPort }}
+          containerPort: {{ .containerPort }}
+          {{- if .hostPort }}
+          hostPort: {{ .hostPort }}
+          {{- end}}
+          protocol: {{ .protocol }}
+        {{- end }}
+        {{- if (and .Values.status.http.enabled .Values.status.enabled)}}
+        - name: status
+          containerPort: {{ .Values.status.http.containerPort }}
+          {{- if .Values.status.http.hostPort }}
+          hostPort: {{ .Values.status.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.status.tls.enabled .Values.status.enabled) }}
+        - name: status-tls
+          containerPort: {{ .Values.status.tls.containerPort }}
+          {{- if .Values.status.tls.hostPort }}
+          hostPort: {{ .Values.status.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.cluster.tls.enabled .Values.cluster.enabled) }}
+        - name: cluster-tls
+          containerPort: {{ .Values.cluster.tls.containerPort }}
+          {{- if .Values.cluster.tls.hostPort }}
+          hostPort: {{ .Values.cluster.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if .Values.enterprise.enabled }}
+        {{- if (and .Values.manager.http.enabled .Values.manager.enabled) }}
+        - name: manager
+          containerPort: {{ .Values.manager.http.containerPort }}
+          {{- if .Values.manager.http.hostPort }}
+          hostPort: {{ .Values.manager.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.manager.tls.enabled .Values.manager.enabled) }}
+        - name: manager-tls
+          containerPort: {{ .Values.manager.tls.containerPort }}
+          {{- if .Values.manager.tls.hostPort }}
+          hostPort: {{ .Values.manager.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.http.enabled .Values.portal.enabled) }}
+        - name: portal
+          containerPort: {{ .Values.portal.http.containerPort }}
+          {{- if .Values.portal.http.hostPort }}
+          hostPort: {{ .Values.portal.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portal.tls.enabled .Values.portal.enabled) }}
+        - name: portal-tls
+          containerPort: {{ .Values.portal.tls.containerPort }}
+          {{- if .Values.portal.tls.hostPort }}
+          hostPort: {{ .Values.portal.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.http.enabled .Values.portalapi.enabled) }}
+        - name: portalapi
+          containerPort: {{ .Values.portalapi.http.containerPort }}
+          {{- if .Values.portalapi.http.hostPort }}
+          hostPort: {{ .Values.portalapi.http.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.portalapi.tls.enabled .Values.portalapi.enabled) }}
+        - name: portalapi-tls
+          containerPort: {{ .Values.portalapi.tls.containerPort }}
+          {{- if .Values.portalapi.tls.hostPort }}
+          hostPort: {{ .Values.portalapi.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- if (and .Values.clustertelemetry.tls.enabled .Values.clustertelemetry.enabled) }}
+        - name: clustert-tls
+          containerPort: {{ .Values.clustertelemetry.tls.containerPort }}
+          {{- if .Values.clustertelemetry.tls.hostPort }}
+          hostPort: {{ .Values.clustertelemetry.tls.hostPort }}
+          {{- end}}
+          protocol: TCP
+        {{- end }}
+        {{- end }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 10 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 10 }}
+        readinessProbe:
+{{ include "kong.proxy.compatibleReadiness" . | indent 10 }}
+        livenessProbe:
+{{ toYaml .Values.livenessProbe | indent 10 }}
+        {{- if .Values.startupProbe }}
+        startupProbe:
+{{ toYaml .Values.startupProbe | indent 10 }}
+        {{- end }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        {{- end }} {{/* End of Kong container spec */}}
+    {{- if .Values.affinity }}
+      affinity:
+{{ toYaml .Values.affinity | indent 8 }}
+    {{- end }}
+    {{- if .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+{{ toYaml .Values.topologySpreadConstraints | indent 8 }}
+    {{- end }}
+      securityContext:
+      {{- include "kong.podsecuritycontext" . | nindent 8 }}
+    {{- if .Values.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.nodeSelector | indent 8 }}
+    {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+    {{- if .Values.tolerations }}
+      tolerations:
+{{ toYaml .Values.tolerations | indent 8 }}
+    {{- end }}
+      volumes:
+      {{- include "kong.volumes" . | nindent 8 -}}
+      {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
+{{- end }}
+*/}}
diff --git a/charts/kong/jail/tmp-kong-deploy.yaml b/charts/kong/jail/tmp-kong-deploy.yaml
new file mode 100644
index 000000000..6675de1c3
--- /dev/null
+++ b/charts/kong/jail/tmp-kong-deploy.yaml
@@ -0,0 +1,36 @@
+{{- if .Values.deployment.kong.enabled }}
+{{- $deployConfig := dict -}}
+{{- $deployConfig = (mustMerge $deployConfig .deployment.kong) -}}
+{{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
+{{- $podConfig := dict -}}
+{{- $_ := set $deployPreRender "name" (printf "%s-%s (include "kong.fullname" .) "proxy") -}}
+{{- $_ := set $deployPreRender "namespace" (include "kong.namespace" .) -}}
+# TODO 921 combine with .deployment.controller.labels.
+# unsure how the types work out here--kong.metaLabels produces YAML output, and I'm not sure if we can
+# just concat them as strings or what. can maybe ignore .deployment.controller.labels--it wasn't
+# included in the past and we instead set extraLabels on EVERYTHING. that's apparently sufficient for
+# user needs (there's been no request to provide separate per-resource labels here), but incongruous
+# with how we handle annotations
+{{- $_ := set $deployPreRender "labels" (include "kong.metaLabels" .) -}}
+# TODO 921 this needs changes to the helper template since we'll need to point to two different sets of pods.
+# in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
+# need to pass a suffix to the helper
+{{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
+{{- $_ := set $deployPreRender "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
+
+{{/*
+# TODO 921 unsure if we want to keep this gate. it's probably simpler to just say "don't set .Values.dblessConfig.config
+# if you're not using a setup that makes sense for it"
+{{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off" )) }}
+*/}}
+{{- if .Values.dblessConfig.config }}
+{{- $- := set $deployPreRender "dblessChecksum" (toYaml .Values.dblessConfig.config | sha256sum) }}
+{{- end }}
+{{/* # TODO 921 end of the old gate above
+{{- end }}
+*/}}
+
+{{- $preRender := dict -}}
+{{- $_ := set $preRender "deployment" $deployPreRender -}}
+{{- $_ := set $preRender "pod" $podPreRender -}}
+{{- $_ := set $deployConfig "pre" $preRender -}} # TODO 921 maybe think of a better key name
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index 611cab615..ca9fe3123 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -33,3 +33,5 @@
 {{- $_ := set $preRender "deployment" $deployPreRender -}}
 {{- $_ := set $preRender "pod" $podPreRender -}}
 {{- $_ := set $deployConfig "pre" $preRender -}} # TODO 921 maybe think of a better key name
+
+{{- template "kong.deployment" $deployConfig -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 832fd37fe..0cff51977 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -137,177 +137,6 @@ spec:
       {{- end }}
       containers: {}
       {{/* CONTAINERS CURRENTLY DISABLED
-      # TODO 921 all sorts of fun stuff in this section. .pod.container
-      # contains generic configuration, but in practice this needs to be
-      # combined with a bunch of app-specific config that lives elsewhere, and
-      # not just in env. a generic function in context is probably quite
-      # difficult, though doable if you provide pass-in points for everything
-      # (mounts, env, god knows what else) and assign a var at the beginning.
-      # probably just use dedicated functions for kong and controller
-      # containers for now, with a TODO to genericize further
-      {{- if .Values.ingressController.enabled }}
-      {{- include "kong.controller-container" . | nindent 6 }}
-      {{ end }}
-      {{- if .Values.deployment.sidecarContainers }}
-      {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
-      {{- end }}
-      # TODO 921 this presumably goes away
-      {{- if .Values.deployment.kong.enabled }}
-      - name: "proxy"
-        image: {{ include "kong.getRepoTag" .Values.image }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        securityContext:
-        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
-        env:
-        {{- include "kong.no_daemon_env" . | nindent 8 }}
-        lifecycle:
-          {{- toYaml .Values.lifecycle | nindent 10 }}
-        ports:
-        {{- if (and .Values.admin.http.enabled .Values.admin.enabled) }}
-        - name: admin
-          containerPort: {{ .Values.admin.http.containerPort }}
-          {{- if .Values.admin.http.hostPort }}
-          hostPort: {{ .Values.admin.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.admin.tls.enabled .Values.admin.enabled) }}
-        - name: admin-tls
-          containerPort: {{ .Values.admin.tls.containerPort }}
-          {{- if .Values.admin.tls.hostPort }}
-          hostPort: {{ .Values.admin.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.proxy.http.enabled .Values.proxy.enabled) }}
-        - name: proxy
-          containerPort: {{ .Values.proxy.http.containerPort }}
-          {{- if .Values.proxy.http.hostPort }}
-          hostPort: {{ .Values.proxy.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.proxy.tls.enabled .Values.proxy.enabled)}}
-        - name: proxy-tls
-          containerPort: {{ .Values.proxy.tls.containerPort }}
-          {{- if .Values.proxy.tls.hostPort }}
-          hostPort: {{ .Values.proxy.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- range .Values.proxy.stream }}
-        - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
-          containerPort: {{ .containerPort }}
-          {{- if .hostPort }}
-          hostPort: {{ .hostPort }}
-          {{- end}}
-          protocol: {{ .protocol }}
-        {{- end }}
-        {{- range .Values.udpProxy.stream }}
-        - name: streamudp-{{ .containerPort }}
-          containerPort: {{ .containerPort }}
-          {{- if .hostPort }}
-          hostPort: {{ .hostPort }}
-          {{- end}}
-          protocol: {{ .protocol }}
-        {{- end }}
-        {{- if (and .Values.status.http.enabled .Values.status.enabled)}}
-        - name: status
-          containerPort: {{ .Values.status.http.containerPort }}
-          {{- if .Values.status.http.hostPort }}
-          hostPort: {{ .Values.status.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.status.tls.enabled .Values.status.enabled) }}
-        - name: status-tls
-          containerPort: {{ .Values.status.tls.containerPort }}
-          {{- if .Values.status.tls.hostPort }}
-          hostPort: {{ .Values.status.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.cluster.tls.enabled .Values.cluster.enabled) }}
-        - name: cluster-tls
-          containerPort: {{ .Values.cluster.tls.containerPort }}
-          {{- if .Values.cluster.tls.hostPort }}
-          hostPort: {{ .Values.cluster.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if .Values.enterprise.enabled }}
-        {{- if (and .Values.manager.http.enabled .Values.manager.enabled) }}
-        - name: manager
-          containerPort: {{ .Values.manager.http.containerPort }}
-          {{- if .Values.manager.http.hostPort }}
-          hostPort: {{ .Values.manager.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.manager.tls.enabled .Values.manager.enabled) }}
-        - name: manager-tls
-          containerPort: {{ .Values.manager.tls.containerPort }}
-          {{- if .Values.manager.tls.hostPort }}
-          hostPort: {{ .Values.manager.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portal.http.enabled .Values.portal.enabled) }}
-        - name: portal
-          containerPort: {{ .Values.portal.http.containerPort }}
-          {{- if .Values.portal.http.hostPort }}
-          hostPort: {{ .Values.portal.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portal.tls.enabled .Values.portal.enabled) }}
-        - name: portal-tls
-          containerPort: {{ .Values.portal.tls.containerPort }}
-          {{- if .Values.portal.tls.hostPort }}
-          hostPort: {{ .Values.portal.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portalapi.http.enabled .Values.portalapi.enabled) }}
-        - name: portalapi
-          containerPort: {{ .Values.portalapi.http.containerPort }}
-          {{- if .Values.portalapi.http.hostPort }}
-          hostPort: {{ .Values.portalapi.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portalapi.tls.enabled .Values.portalapi.enabled) }}
-        - name: portalapi-tls
-          containerPort: {{ .Values.portalapi.tls.containerPort }}
-          {{- if .Values.portalapi.tls.hostPort }}
-          hostPort: {{ .Values.portalapi.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.clustertelemetry.tls.enabled .Values.clustertelemetry.enabled) }}
-        - name: clustert-tls
-          containerPort: {{ .Values.clustertelemetry.tls.containerPort }}
-          {{- if .Values.clustertelemetry.tls.hostPort }}
-          hostPort: {{ .Values.clustertelemetry.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- end }}
-        volumeMounts:
-        {{- include "kong.volumeMounts" . | nindent 10 }}
-        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 10 }}
-        readinessProbe:
-{{ include "kong.proxy.compatibleReadiness" . | indent 10 }}
-        livenessProbe:
-{{ toYaml .Values.livenessProbe | indent 10 }}
-        {{- if .Values.startupProbe }}
-        startupProbe:
-{{ toYaml .Values.startupProbe | indent 10 }}
-        {{- end }}
-        resources:
-{{ toYaml .Values.resources | indent 10 }}
-        {{- end }} {{/* TODO 921 End of Kong container spec */}}
-CONTAINERS CURRENTLY DISABLED */}}
     {{- if .Values.affinity }}
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}
@@ -329,315 +158,7 @@ CONTAINERS CURRENTLY DISABLED */}}
 {{ toYaml .pod.tolerations | indent 8 }}
     {{- end }}
       volumes:
-      # TODO 921 pass in
       {{- .pre.pod.volumes | nindent 8 -}}
       {{- .pre.pod.userVolumes | nindent 8 -}}
 
 {{- end -}} {{/* end kong.deployment */}}
-
-{{- if .Values.deployment.controller.enabled }}
-apiVersion: apps/v1
-{{- if .Values.deployment.daemonset }}
-kind: DaemonSet
-{{- else }}
-kind: Deployment
-{{- end }}
-metadata:
-  name: {{ template "kong.fullname" . }}
-  namespace:  {{ template "kong.namespace" . }}
-  labels:
-    {{- include "kong.metaLabels" . | nindent 4 }}
-    app.kubernetes.io/component: app
-  {{- if .Values.deploymentAnnotations }}
-  annotations:
-  {{- range $key, $value := .Values.deploymentAnnotations }}
-    {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
-  {{- end }}
-  {{- end }}
-spec:
-  {{- if not .Values.autoscaling.enabled }}
-  {{- if not .Values.deployment.daemonset }}
-  replicas: {{ .Values.replicaCount }}
-  {{- end }}
-  {{- end }}
-  selector:
-    matchLabels:
-      {{- include "kong.selectorLabels" . | nindent 6 }}
-  {{- if .Values.updateStrategy }}
-  {{- if .Values.deployment.daemonset }}
-  updateStrategy:
-  {{- else }}
-  strategy:
-  {{- end }}
-{{ toYaml .Values.updateStrategy | indent 4 }}
-  {{- end }}
-  {{- if .Values.deployment.minReadySeconds }}
-  minReadySeconds: {{ .Values.deployment.minReadySeconds }}
-  {{- end }}
-
-  template:
-    metadata:
-      annotations:
-        {{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
-        kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
-        {{- end }}
-        {{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off" )) }}
-        {{- if .Values.dblessConfig.config }}
-        checksum/dbless.config: {{ toYaml .Values.dblessConfig.config | sha256sum }}
-        {{- end }}
-        {{- end }}
-        {{- if .Values.podAnnotations }}
-        {{- range $key, $value := .Values.podAnnotations }}
-        {{ $key }}: {{ include "kong.renderTpl" (dict "value" $value "context" $) | quote }}
-        {{- end }}
-        {{- end }}
-      labels:
-        {{- include "kong.metaLabels" . | nindent 8 }}
-        app.kubernetes.io/component: app
-        app: {{ template "kong.fullname" . }}
-        version: {{ .Chart.AppVersion | quote }}
-        {{- if .Values.podLabels }}
-        {{ include "kong.renderTpl" (dict "value" .Values.podLabels "context" $) | nindent 8 }}
-        {{- end }}
-    spec:
-      {{- if .Values.deployment.hostNetwork }}
-      hostNetwork: true
-      {{- end }}
-      {{- if .Values.priorityClassName }}
-      priorityClassName: "{{ .Values.priorityClassName }}"
-      {{- end }}
-      {{- if or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name }}
-      serviceAccountName: {{ template "kong.serviceAccountName" . }}
-      {{- end }}
-      {{- if (and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) .Values.deployment.serviceAccount.automountServiceAccountToken) }}
-      automountServiceAccountToken: true
-      {{- else }}
-      automountServiceAccountToken: false
-      {{ end }}
-      {{- if .Values.image.pullSecrets }}
-      imagePullSecrets:
-      {{- range .Values.image.pullSecrets }}
-        - name: {{ . }}
-      {{- end }}
-      {{- end }}
-      {{- if .Values.deployment.kong.enabled }}
-      initContainers:
-      - name: clear-stale-pid
-        image: {{ include "kong.getRepoTag" .Values.image }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        securityContext:
-        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
-        resources:
-{{ toYaml .Values.resources | indent 10 }}
-        command:
-        - "rm"
-        - "-vrf"
-        - "$KONG_PREFIX/pids"
-        env:
-        {{- include "kong.env" . | nindent 8 }}
-        volumeMounts:
-        {{- include "kong.volumeMounts" . | nindent 8 }}
-        {{- if .Values.deployment.initContainers }}
-        {{- toYaml .Values.deployment.initContainers | nindent 6 }}
-        {{- end }}
-        {{- if (and (not (eq .Values.env.database "off")) .Values.waitImage.enabled) }}
-        {{- include "kong.wait-for-db" . | nindent 6 }}
-        {{- end }}
-      {{- end }}
-      {{- if .Values.deployment.hostAliases }}
-      hostAliases:
-        {{- toYaml .Values.deployment.hostAliases | nindent 6 }}
-      {{- end}}
-      {{- if .Values.dnsPolicy }}
-      dnsPolicy: {{ .Values.dnsPolicy | quote }}
-      {{- end }}
-      {{- if .Values.dnsConfig }}
-      dnsConfig:
-{{ toYaml .Values.dnsConfig | indent 8 }}
-      {{- end }}
-      containers:
-      {{- if .Values.ingressController.enabled }}
-      {{- include "kong.controller-container" . | nindent 6 }}
-      {{ end }}
-      {{- if .Values.deployment.sidecarContainers }}
-      {{- toYaml .Values.deployment.sidecarContainers | nindent 6 }}
-      {{- end }}
-      {{- if .Values.deployment.kong.enabled }}
-      - name: "proxy"
-        image: {{ include "kong.getRepoTag" .Values.image }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        securityContext:
-        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
-        env:
-        {{- include "kong.no_daemon_env" . | nindent 8 }}
-        lifecycle:
-          {{- toYaml .Values.lifecycle | nindent 10 }}
-        ports:
-        {{- if (and .Values.admin.http.enabled .Values.admin.enabled) }}
-        - name: admin
-          containerPort: {{ .Values.admin.http.containerPort }}
-          {{- if .Values.admin.http.hostPort }}
-          hostPort: {{ .Values.admin.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.admin.tls.enabled .Values.admin.enabled) }}
-        - name: admin-tls
-          containerPort: {{ .Values.admin.tls.containerPort }}
-          {{- if .Values.admin.tls.hostPort }}
-          hostPort: {{ .Values.admin.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.proxy.http.enabled .Values.proxy.enabled) }}
-        - name: proxy
-          containerPort: {{ .Values.proxy.http.containerPort }}
-          {{- if .Values.proxy.http.hostPort }}
-          hostPort: {{ .Values.proxy.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.proxy.tls.enabled .Values.proxy.enabled)}}
-        - name: proxy-tls
-          containerPort: {{ .Values.proxy.tls.containerPort }}
-          {{- if .Values.proxy.tls.hostPort }}
-          hostPort: {{ .Values.proxy.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- range .Values.proxy.stream }}
-        - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
-          containerPort: {{ .containerPort }}
-          {{- if .hostPort }}
-          hostPort: {{ .hostPort }}
-          {{- end}}
-          protocol: {{ .protocol }}
-        {{- end }}
-        {{- range .Values.udpProxy.stream }}
-        - name: streamudp-{{ .containerPort }}
-          containerPort: {{ .containerPort }}
-          {{- if .hostPort }}
-          hostPort: {{ .hostPort }}
-          {{- end}}
-          protocol: {{ .protocol }}
-        {{- end }}
-        {{- if (and .Values.status.http.enabled .Values.status.enabled)}}
-        - name: status
-          containerPort: {{ .Values.status.http.containerPort }}
-          {{- if .Values.status.http.hostPort }}
-          hostPort: {{ .Values.status.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.status.tls.enabled .Values.status.enabled) }}
-        - name: status-tls
-          containerPort: {{ .Values.status.tls.containerPort }}
-          {{- if .Values.status.tls.hostPort }}
-          hostPort: {{ .Values.status.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.cluster.tls.enabled .Values.cluster.enabled) }}
-        - name: cluster-tls
-          containerPort: {{ .Values.cluster.tls.containerPort }}
-          {{- if .Values.cluster.tls.hostPort }}
-          hostPort: {{ .Values.cluster.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if .Values.enterprise.enabled }}
-        {{- if (and .Values.manager.http.enabled .Values.manager.enabled) }}
-        - name: manager
-          containerPort: {{ .Values.manager.http.containerPort }}
-          {{- if .Values.manager.http.hostPort }}
-          hostPort: {{ .Values.manager.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.manager.tls.enabled .Values.manager.enabled) }}
-        - name: manager-tls
-          containerPort: {{ .Values.manager.tls.containerPort }}
-          {{- if .Values.manager.tls.hostPort }}
-          hostPort: {{ .Values.manager.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portal.http.enabled .Values.portal.enabled) }}
-        - name: portal
-          containerPort: {{ .Values.portal.http.containerPort }}
-          {{- if .Values.portal.http.hostPort }}
-          hostPort: {{ .Values.portal.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portal.tls.enabled .Values.portal.enabled) }}
-        - name: portal-tls
-          containerPort: {{ .Values.portal.tls.containerPort }}
-          {{- if .Values.portal.tls.hostPort }}
-          hostPort: {{ .Values.portal.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portalapi.http.enabled .Values.portalapi.enabled) }}
-        - name: portalapi
-          containerPort: {{ .Values.portalapi.http.containerPort }}
-          {{- if .Values.portalapi.http.hostPort }}
-          hostPort: {{ .Values.portalapi.http.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.portalapi.tls.enabled .Values.portalapi.enabled) }}
-        - name: portalapi-tls
-          containerPort: {{ .Values.portalapi.tls.containerPort }}
-          {{- if .Values.portalapi.tls.hostPort }}
-          hostPort: {{ .Values.portalapi.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- if (and .Values.clustertelemetry.tls.enabled .Values.clustertelemetry.enabled) }}
-        - name: clustert-tls
-          containerPort: {{ .Values.clustertelemetry.tls.containerPort }}
-          {{- if .Values.clustertelemetry.tls.hostPort }}
-          hostPort: {{ .Values.clustertelemetry.tls.hostPort }}
-          {{- end}}
-          protocol: TCP
-        {{- end }}
-        {{- end }}
-        volumeMounts:
-        {{- include "kong.volumeMounts" . | nindent 10 }}
-        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 10 }}
-        readinessProbe:
-{{ include "kong.proxy.compatibleReadiness" . | indent 10 }}
-        livenessProbe:
-{{ toYaml .Values.livenessProbe | indent 10 }}
-        {{- if .Values.startupProbe }}
-        startupProbe:
-{{ toYaml .Values.startupProbe | indent 10 }}
-        {{- end }}
-        resources:
-{{ toYaml .Values.resources | indent 10 }}
-        {{- end }} {{/* End of Kong container spec */}}
-    {{- if .Values.affinity }}
-      affinity:
-{{ toYaml .Values.affinity | indent 8 }}
-    {{- end }}
-    {{- if .Values.topologySpreadConstraints }}
-      topologySpreadConstraints:
-{{ toYaml .Values.topologySpreadConstraints | indent 8 }}
-    {{- end }}
-      securityContext:
-      {{- include "kong.podsecuritycontext" . | nindent 8 }}
-    {{- if .Values.nodeSelector }}
-      nodeSelector:
-{{ toYaml .Values.nodeSelector | indent 8 }}
-    {{- end }}
-      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
-    {{- if .Values.tolerations }}
-      tolerations:
-{{ toYaml .Values.tolerations | indent 8 }}
-    {{- end }}
-      volumes:
-      {{- include "kong.volumes" . | nindent 8 -}}
-      {{- include "kong.userDefinedVolumes" . | nindent 8 -}}
-{{- end }}
-

From d91dea824e6fe18ef047cef342da7b920b2a0b38 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 15:31:15 -0800
Subject: [PATCH 16/24] wip: fix template errors

---
 charts/kong/templates/_helpers.tpl               |  7 -------
 charts/kong/templates/controller-deploy.yaml     | 11 ++++++-----
 charts/kong/templates/deployment-controller.yaml |  1 -
 3 files changed, 6 insertions(+), 13 deletions(-)

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index 04f2b678c..223266929 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -1001,13 +1001,6 @@ valueFrom:
     key: {{ .key }}
 {{- end -}}
 
-{{- define "secretkeyref" -}}
-valueFrom:
-  secretKeyRef:
-    name: {{ .name }}
-    key: {{ .key }}
-{{- end -}}
-
 {{- define "kong.no_daemon_env" -}}
 {{- template "kong.env" . }}
 - name: KONG_NGINX_DAEMON
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index ca9fe3123..acef4e70e 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -4,7 +4,7 @@
 {{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
 {{- $podPreRender := dict -}}
 
-{{- $_ := set $deployPreRender "name" (printf "%s-%s (include "kong.fullname" .) "controller") -}}
+{{- $_ := set $deployPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "controller") -}}
 {{- $_ := set $deployPreRender "namespace" (include "kong.namespace" .) -}}
 # TODO 921 combine with .deployment.controller.labels.
 # unsure how the types work out here--kong.metaLabels produces YAML output, and I'm not sure if we can
@@ -17,16 +17,16 @@
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
-{{- $_ := set $deployPreRender "serviceAccountTokenName" (template "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 # TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
 # pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
 # unnecessary volumes doesn't really hurt us
-{{- $_ := set $deployPreRender "volumes" (template "kong.volumes" .) -}}
-{{- $_ := set $deployPreRender "userVolumes" (template "kong.userDefinedVolumes" .) -}}
+{{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
+{{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "labels" (include "kong.metaLabels" .) -}}
-{{- $_ := set $podPreRender "name" (printf "%s-%s (include "kong.fullname" .) "controller" -}}
+{{- $_ := set $podPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "controller") -}}
 {{- $_ := set $podPreRender "versionLabel" .Chart.AppVersion -}}
 
 {{- $preRender := dict -}}
@@ -35,3 +35,4 @@
 {{- $_ := set $deployConfig "pre" $preRender -}} # TODO 921 maybe think of a better key name
 
 {{- template "kong.deployment" $deployConfig -}}
+{{- end -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 0cff51977..f693c9cf9 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -136,7 +136,6 @@ spec:
 {{ toYaml .pod.dnsConfig | indent 8 }}
       {{- end }}
       containers: {}
-      {{/* CONTAINERS CURRENTLY DISABLED
     {{- if .Values.affinity }}
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}

From f1145ab62afd72e784feb6fbca65bdf3a4e896f7 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 16:06:00 -0800
Subject: [PATCH 17/24] wip: successful render

---
 charts/kong/templates/controller-deploy.yaml  |  2 +-
 .../kong/templates/deployment-controller.yaml | 26 ++++++++++---------
 charts/kong/values.yaml                       |  1 +
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index acef4e70e..994418b4d 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -1,6 +1,6 @@
 {{- if .Values.deployment.controller.enabled }}
 {{- $deployConfig := dict -}}
-{{- $deployConfig = (mustMerge $deployConfig .deployment.controller) -}}
+{{- $deployConfig = (mustMerge $deployConfig .Values.deployment.controller) -}}
 {{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
 {{- $podPreRender := dict -}}
 
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index f693c9cf9..ad420e10f 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -1,8 +1,9 @@
 {{- define "kong.deployment" -}}
-apiVersion: apps/v1
 {{- if .daemonset }}
+apiVersion: apps/v1
 kind: DaemonSet
 {{- else }}
+apiVersion: apps/v1
 kind: Deployment
 {{- end }}
 metadata:
@@ -25,7 +26,7 @@ spec:
   {{- end }}
   selector:
     matchLabels:
-      {{- .pre.deployment.selectorMatchLabels }}
+{{ .pre.deployment.selectorMatchLabels | nindent 6 -}}
   {{- if .updateStrategy }}
   {{- if .daemonset }}
   updateStrategy:
@@ -43,12 +44,12 @@ spec:
       annotations:
         {{/* # TODO 921 some keys in the pod template end up being from the deployment dict, so probably no separate define 
              # there's no obvious reason to separate them, since there can only be one per Deployment anyway
-        */}}
-        {{- if (and (not .serviceAccount.automountServiceAccountToken) (or .serviceAccount.create .serviceAccount.name)) }}
+        */ -}}
+        {{- if (and (not .pod.serviceAccount.automountServiceAccountToken) (or .pod.serviceAccount.create .pod.serviceAccount.name)) }}
         {{/* # TODO 921 needs to be passed in as-is. possibly should be made relative, since the generator will attempt to use the same
              # name, but lives in a separate service-account.yaml template, so vov
              # TODO above is complete but keeping the note around for now
-        */}}
+        */ -}}
         kuma.io/service-account-token-volume: {{ .pre.deployment.serviceAccountTokenName }}
         {{- end }}
         {{/*  # TODO 921 special to DB-less Kong with static config. I think this can be handled with the .Values.dblessConfig.config only:
@@ -78,10 +79,10 @@ spec:
       {{- if .pod.priorityClassName }}
       priorityClassName: "{{ .pod.priorityClassName }}"
       {{- end }}
-      {{- if or .serviceAccount.create .serviceAccount.name }}
+      {{- if or .pod.serviceAccount.create .pod.serviceAccount.name }}
       serviceAccountName: {{ .pre.deployment.serviceAccountTokenName }}
       {{- end }}
-      {{- if (and (or .serviceAccount.create .serviceAccount.name) .serviceAccount.automountServiceAccountToken) }}
+      {{- if (and (or .pod.serviceAccount.create .pod.serviceAccount.name) .pod.serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
       {{- else }}
       automountServiceAccountToken: false
@@ -100,6 +101,7 @@ spec:
       # does need Kong's image, security context, resources, env, so probably
       # create a dedicated helper to create the initContainer and then support
       # injecting initContainers in general. .Values stuff left as-is from here
+{{/*
       {{- if .Values.deployment.kong.enabled }}
       initContainers:
       - name: clear-stale-pid
@@ -124,6 +126,7 @@ spec:
         {{- include "kong.wait-for-db" . | nindent 6 }}
         {{- end }}
       {{- end }} # TODO 921 END initContainers clear-stale-pid
+*/}}
       {{- if .pod.hostAliases }}
       hostAliases:
         {{- toYaml .pod.hostAliases | nindent 6 }}
@@ -135,8 +138,8 @@ spec:
       dnsConfig:
 {{ toYaml .pod.dnsConfig | indent 8 }}
       {{- end }}
-      containers: {}
-    {{- if .Values.affinity }}
+      containers: []
+    {{- if .pod.affinity }}
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}
     {{- end }}
@@ -145,7 +148,6 @@ spec:
 {{ toYaml .pod.topologySpreadConstraints | indent 8 }}
     {{- end }}
       securityContext:
-      # TODO 921 this previously used a helper but it only did the below and seemed kinda useless
       {{- .pod.securityContext | toYaml | nindent 8 }}
     {{- if .pod.nodeSelector }}
       nodeSelector:
@@ -157,7 +159,7 @@ spec:
 {{ toYaml .pod.tolerations | indent 8 }}
     {{- end }}
       volumes:
-      {{- .pre.pod.volumes | nindent 8 -}}
-      {{- .pre.pod.userVolumes | nindent 8 -}}
+      {{- .pre.deployment.volumes | nindent 8 -}}
+      {{- .pre.deployment.userVolumes | nindent 8 -}}
 
 {{- end -}} {{/* end kong.deployment */}}
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 607876ac4..54f6896a7 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -56,6 +56,7 @@
 # controller Deployment, but `.podAnnotations` for the Kong Deployment's Pods.
 deployment:
   controller:
+    enabled: true
     labels: {}
     annotations: {} # TODO 921 add any defaults
     # TODO 921 the equivalent root setting has "default" values: https://github.com/Kong/charts/blob/kong-2.30.0/charts/kong/values.yaml#L887-L904

From 65bf666b8aefe72968121dd2926d8875f4cd1823 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 7 Nov 2023 17:42:48 -0800
Subject: [PATCH 18/24] wip: working controller deploy

---
 charts/kong/templates/_helpers.tpl            | 29 +++++-----
 charts/kong/templates/controller-deploy.yaml  | 15 +++++
 .../kong/templates/deployment-controller.yaml |  4 +-
 charts/kong/templates/service-account.yaml    |  2 +-
 charts/kong/values.yaml                       | 57 ++++++++++++++++++-
 5 files changed, 87 insertions(+), 20 deletions(-)

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index 223266929..cef9b17c7 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -534,7 +534,7 @@ The name of the Service which will be used by the controller to update the Ingre
 */}}
 
 {{- $userEnv := dict -}}
-{{- range $key, $val := .Values.ingressController.env }}
+{{- range $key, $val := .Values.deployment.controller.pod.container.env }}
   {{- $upper := upper $key -}}
   {{- $var := printf "CONTROLLER_%s" $upper -}}
   {{- $_ := set $userEnv $var $val -}}
@@ -714,7 +714,7 @@ The name of the Service which will be used by the controller to update the Ingre
 {{- end -}}
 
 {{- define "controller.adminApiCertVolumeMount" -}}
-{{- if and .Values.ingressController.adminApi.tls.client.enabled .Values.ingressController.enabled }}
+{{- if .adminApi.tls.client.enabled }}
 - name: admin-api-cert
   mountPath: /etc/secrets/admin-api-cert
   readOnly: true
@@ -934,11 +934,9 @@ The name of the Service which will be used by the controller to update the Ingre
   {{- end }}
   {{- end }}
   ports:
-  # TODO 921 would not have global value here, need to pass something in. can
-  # pass in port or null if none--it's the only value here
-  {{- if .Values.ingressController.admissionWebhook.enabled }}
+  {{- if .pre.admissionWebhook.enabled }}
   - name: webhook
-    containerPort: {{ .Values.ingressController.admissionWebhook.port }}
+    containerPort: {{ .pre.admissionWebhook.port }}
     protocol: TCP
   {{- end }}
   {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .image)) -}}
@@ -958,8 +956,10 @@ The name of the Service which will be used by the controller to update the Ingre
         apiVersion: v1
         fieldPath: metadata.namespace
 # TODO 921 this is not currently designed to be at all relative and relies on
-# external sections
-{{- include "kong.ingressController.env" .  | indent 2 }}
+# external sections. the template invoker renders this from the root and passes
+# it in as .env
+  env:
+{{- .pre.env  | indent 4 }}
   image: {{ include "kong.getRepoTag" .image }}
   imagePullPolicy: {{ .image.pullPolicy }}
 {{/* disableReadiness is a hidden setting to drop this block entirely for use with a debugger
@@ -967,7 +967,7 @@ The name of the Service which will be used by the controller to update the Ingre
      check type, and all HTTP checks freeze when a debugger pauses operation.
      Setting disableReadiness to ANY value disables the probes.
 */}}
-{{- if (not (hasKey .Values.ingressController "disableProbes")) }}
+{{- if (not (hasKey .pre "disableProbes")) }}
   readinessProbe:
 {{ toYaml .readinessProbe | indent 4 }}
   livenessProbe:
@@ -976,22 +976,19 @@ The name of the Service which will be used by the controller to update the Ingre
   resources:
 {{ toYaml .resources | indent 4 }}
   volumeMounts:
-# TODO 921 comes from external, but only needs the single boolean in the
-# initial if
-{{- if .ingressController.admissionWebhook.enabled }}
+{{- if .pre.admissionWebhook.enabled }}
   - name: webhook-cert
     mountPath: /admission-webhook
     readOnly: true
 {{- end }}
-# TODO 921 would not be in the container context
-{{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
-  - name: {{ template "kong.serviceAccountTokenName" . }}
+{{- if (and (not .pre.serviceAccount.automountServiceAccountToken) (or .pre.serviceAccount.create .pre.serviceAccount.name)) }}
+  - name: {{ .pre.serviceAccountTokenName }}
     mountPath: /var/run/secrets/kubernetes.io/serviceaccount
     readOnly: true
 {{- end }}
   {{- include "kong.userDefinedVolumeMounts" . | nindent 2 }}
   # TODO 921 relies on some content under .ingressController
-  {{- include "controller.adminApiCertVolumeMount" . | nindent 2 }}
+  {{- include "controller.adminApiCertVolumeMount" .pre | nindent 2 }}
 {{- end -}}
 
 {{- define "secretkeyref" -}}
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index 994418b4d..11ba29941 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -18,6 +18,7 @@
 # need to pass a suffix to the helper
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
 # TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
 # pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
 # unnecessary volumes doesn't really hurt us
@@ -29,6 +30,20 @@
 {{- $_ := set $podPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "controller") -}}
 {{- $_ := set $podPreRender "versionLabel" .Chart.AppVersion -}}
 
+# TODO 921 probably use the same pre pattern for this
+{{- $containerConfig := .Values.deployment.controller.pod.container -}}
+{{- $containerPre := dict -}}
+{{- $_ := set $containerPre "admissionWebhook" .Values.admissionWebhook -}}
+{{- $_ := set $containerPre "env" (include "kong.ingressController.env" .) -}}
+{{- $_ := set $containerPre "disableProbes" .Values.ingressController.disableProbes -}}
+{{- $_ := set $containerPre "adminApi" .Values.ingressController.adminApi -}}
+{{- $_ := set $containerPre "serviceAccount" .Values.deployment.controller.pod.serviceAccount -}}
+{{- $_ := set $containerPre "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $containerPre "serviceAccountName" (include "kong.serviceAccountName" .) -}}
+{{- $_ := set $containerConfig "pre" $containerPre -}}
+
+{{- $_ := set $podPreRender "container" (include "kong.controller-container-new" $containerConfig ) -}}
+
 {{- $preRender := dict -}}
 {{- $_ := set $preRender "deployment" $deployPreRender -}}
 {{- $_ := set $preRender "pod" $podPreRender -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index ad420e10f..31603cb10 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -80,7 +80,7 @@ spec:
       priorityClassName: "{{ .pod.priorityClassName }}"
       {{- end }}
       {{- if or .pod.serviceAccount.create .pod.serviceAccount.name }}
-      serviceAccountName: {{ .pre.deployment.serviceAccountTokenName }}
+      serviceAccountName: {{ .pre.deployment.serviceAccountName }}
       {{- end }}
       {{- if (and (or .pod.serviceAccount.create .pod.serviceAccount.name) .pod.serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
@@ -138,7 +138,7 @@ spec:
       dnsConfig:
 {{ toYaml .pod.dnsConfig | indent 8 }}
       {{- end }}
-      containers: []
+      containers: {{ .pre.pod.container | nindent 8 }}
     {{- if .pod.affinity }}
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}
diff --git a/charts/kong/templates/service-account.yaml b/charts/kong/templates/service-account.yaml
index 41ef6ace6..68afcf031 100644
--- a/charts/kong/templates/service-account.yaml
+++ b/charts/kong/templates/service-account.yaml
@@ -1,4 +1,4 @@
-{{- if and (or .Values.deployment.kong.enabled .Values.ingressController.enabled) .Values.deployment.serviceAccount.create -}}
+{{- if and (or .Values.deployment.kong.enabled .Values.deployment.controller.enabled) .Values.deployment.controller.pod.serviceAccount.create -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 54f6896a7..c63ce3715 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -81,6 +81,7 @@ deployment:
     initContainers: []
     pod:
       annotations: {} # .podAnnotations, for the controller
+      # TODO 921 this should probably go elsewhere
       serviceAccount:
         create: true
         # TODO 921 we arguably don't need this with a split controller
@@ -108,6 +109,7 @@ deployment:
       nodeSelector: {}
       securityContext: {}
       container:
+        args: []
         env:
           # The controller disables TLS verification by default because Kong
           # generates self-signed certificates by default. Set this to false once you
@@ -128,7 +130,37 @@ deployment:
           # such as when you are using a "next" tag. Set this to the effective semantic
           # version of your tag: for example if using a "next" image for an unreleased 3.1.0
           # version, set this to "3.1.0".
-          effectiveSemver:
+          effectiveSemver: ""
+        livenessProbe:
+          httpGet:
+            path: "/healthz"
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: "/readyz"
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+          failureThreshold: 3
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          runAsUser: 1000
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+            - ALL
         userDefinedVolumeMounts: []
         # If you want to specify resources, uncomment the following
         # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
@@ -203,6 +235,29 @@ deployment:
 # This may not be desirable when using this chart as a dependency.
 # namespace: "example"
 
+# -----------------------------------------------------------------------------
+# Admission webhook
+# -----------------------------------------------------------------------------
+
+admissionWebhook:
+  enabled: true
+  failurePolicy: Ignore
+  port: 8080
+  certificate:
+    provided: false
+  namespaceSelector: {}
+  # Specifiy the secretName when the certificate is provided via a TLS secret
+  # secretName: ""
+  # Specifiy the CA bundle of the provided certificate.
+  # This is a PEM encoded CA bundle which will be used to validate the webhook certificate. If unspecified, system trust roots on the apiserver are used.
+  # caBundle:
+  #   | Add the CA bundle content here.
+  service:
+    # Specify custom labels for the validation webhook service.
+    labels: {}
+  # Tune the default Kubernetes timeoutSeconds of 10 seconds
+  # timeoutSeconds: 10
+
 # -----------------------------------------------------------------------------
 # Kong parameters
 # -----------------------------------------------------------------------------

From f0ecf72ef2302df419c568b8ce2d196aa8e7adc7 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Thu, 9 Nov 2023 17:47:23 -0800
Subject: [PATCH 19/24] wip: broken kong deploy from helper

Sorta deploy Kong using the Deployment helper. Missing several things.
Selectors are broken as heck and don't work. Need to split that for
separate Deployments better.
---
 charts/kong/templates/_helpers.tpl           | 171 ++++++++++++++++++-
 charts/kong/templates/controller-deploy.yaml |   5 +-
 charts/kong/templates/kong-deploy.yaml       |  68 ++++++++
 charts/kong/values.yaml                      | 121 +++++++++++++
 4 files changed, 362 insertions(+), 3 deletions(-)
 create mode 100644 charts/kong/templates/kong-deploy.yaml

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index cef9b17c7..56cbb9789 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -38,7 +38,13 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 
 {{- define "kong.selectorLabels" -}}
 app.kubernetes.io/name: {{ template "kong.name" . }}
-app.kubernetes.io/component: app
+app.kubernetes.io/component: proxy
+app.kubernetes.io/instance: "{{ .Release.Name }}"
+{{- end -}}
+
+{{- define "kong.controllerSelectorLabels" -}}
+app.kubernetes.io/name: {{ template "kong.name" . }}
+app.kubernetes.io/component: controller
 app.kubernetes.io/instance: "{{ .Release.Name }}"
 {{- end -}}
 
@@ -923,6 +929,169 @@ The name of the Service which will be used by the controller to update the Ingre
   {{- include "controller.adminApiCertVolumeMount" . | nindent 2 }}
 {{- end -}}
 
+{{- define "kong.proxy-container-new" -}}
+- name: "proxy"
+  image: {{ include "kong.getRepoTag" .image }}
+  imagePullPolicy: {{ .image.pullPolicy }}
+  securityContext:
+  {{ toYaml .securityContext | nindent 4 }}
+{{/*   TODO 921 this helper still pulls from the old .Values.env. We need to pass in the deployment.kong.pod.container.env
+  env:
+  {{- (include "kong.no_daemon_env" .) | nindent 2 }}
+*/}}
+  lifecycle:
+    {{- toYaml .lifecycle | nindent 4 }}
+  ports:
+  {{- if (and .pre.svc.admin.http.enabled .pre.svc.admin.enabled) }}
+  - name: admin
+    containerPort: {{ .pre.svc.admin.http.containerPort }}
+    {{- if .pre.svc.admin.http.hostPort }}
+    hostPort: {{ .pre.svc.admin.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.admin.tls.enabled .pre.svc.admin.enabled) }}
+  - name: admin-tls
+    containerPort: {{ .pre.svc.admin.tls.containerPort }}
+    {{- if .pre.svc.admin.tls.hostPort }}
+    hostPort: {{ .pre.svc.admin.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.proxy.http.enabled .pre.svc.proxy.enabled) }}
+  - name: proxy
+    containerPort: {{ .pre.svc.proxy.http.containerPort }}
+    {{- if .pre.svc.proxy.http.hostPort }}
+    hostPort: {{ .pre.svc.proxy.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.proxy.tls.enabled .pre.svc.proxy.enabled)}}
+  - name: proxy-tls
+    containerPort: {{ .pre.svc.proxy.tls.containerPort }}
+    {{- if .pre.svc.proxy.tls.hostPort }}
+    hostPort: {{ .pre.svc.proxy.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- range .pre.svc.proxy.stream }}
+  - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
+    containerPort: {{ .containerPort }}
+    {{- if .hostPort }}
+    hostPort: {{ .hostPort }}
+    {{- end}}
+    protocol: {{ .protocol }}
+  {{- end }}
+  {{- range .pre.svc.udpProxy.stream }}
+  - name: streamudp-{{ .containerPort }}
+    containerPort: {{ .containerPort }}
+    {{- if .hostPort }}
+    hostPort: {{ .hostPort }}
+    {{- end}}
+    protocol: {{ .protocol }}
+  {{- end }}
+  {{- if (and .pre.svc.status.http.enabled .pre.svc.status.enabled)}}
+  - name: status
+    containerPort: {{ .pre.svc.status.http.containerPort }}
+    {{- if .pre.svc.status.http.hostPort }}
+    hostPort: {{ .pre.svc.status.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.status.tls.enabled .pre.svc.status.enabled) }}
+  - name: status-tls
+    containerPort: {{ .pre.svc.status.tls.containerPort }}
+    {{- if .pre.svc.status.tls.hostPort }}
+    hostPort: {{ .pre.svc.status.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.cluster.tls.enabled .pre.svc.cluster.enabled) }}
+  - name: cluster-tls
+    containerPort: {{ .pre.svc.cluster.tls.containerPort }}
+    {{- if .pre.svc.cluster.tls.hostPort }}
+    hostPort: {{ .pre.svc.cluster.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if .pre.enterprise.enabled }}
+  {{- if (and .pre.svc.manager.http.enabled .pre.svc.manager.enabled) }}
+  - name: manager
+    containerPort: {{ .pre.svc.manager.http.containerPort }}
+    {{- if .pre.svc.manager.http.hostPort }}
+    hostPort: {{ .pre.svc.manager.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.manager.tls.enabled .pre.svc.manager.enabled) }}
+  - name: manager-tls
+    containerPort: {{ .pre.svc.manager.tls.containerPort }}
+    {{- if .pre.svc.manager.tls.hostPort }}
+    hostPort: {{ .pre.svc.manager.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portal.http.enabled .pre.svc.portal.enabled) }}
+  - name: portal
+    containerPort: {{ .pre.svc.portal.http.containerPort }}
+    {{- if .pre.svc.portal.http.hostPort }}
+    hostPort: {{ .pre.svc.portal.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portal.tls.enabled .pre.svc.portal.enabled) }}
+  - name: portal-tls
+    containerPort: {{ .pre.svc.portal.tls.containerPort }}
+    {{- if .pre.svc.portal.tls.hostPort }}
+    hostPort: {{ .pre.svc.portal.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portalapi.http.enabled .pre.svc.portalapi.enabled) }}
+  - name: portalapi
+    containerPort: {{ .pre.svc.portalapi.http.containerPort }}
+    {{- if .pre.svc.portalapi.http.hostPort }}
+    hostPort: {{ .pre.svc.portalapi.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portalapi.tls.enabled .pre.svc.portalapi.enabled) }}
+  - name: portalapi-tls
+    containerPort: {{ .pre.svc.portalapi.tls.containerPort }}
+    {{- if .pre.svc.portalapi.tls.hostPort }}
+    hostPort: {{ .pre.svc.portalapi.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.clustertelemetry.tls.enabled .pre.svc.clustertelemetry.enabled) }}
+  - name: clustert-tls
+    containerPort: {{ .pre.svc.clustertelemetry.tls.containerPort }}
+    {{- if .pre.svc.clustertelemetry.tls.hostPort }}
+    hostPort: {{ .pre.svc.clustertelemetry.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- end }}
+{{/* TODO 921 whatever we're doing for volume mounts
+  volumeMounts:
+  {{- include "kong.volumeMounts" . | nindent 4 }}
+  {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 4 }}
+*/}}
+  readinessProbe:
+{{ toYaml .readinessProbe | indent 4 }}
+{{/* TODO 921 this requires controller-related keys we aren't passing in yet
+{{ include "kong.proxy.compatibleReadiness" . | indent 4 }}
+*/}}
+  livenessProbe:
+{{ toYaml .livenessProbe | indent 4 }}
+  {{- if .startupProbe }}
+  startupProbe:
+{{ toYaml .startupProbe | indent 4 }}
+  {{- end }}
+  resources:
+{{ toYaml .resources | indent 4 }}
+{{- end -}}
+
 {{- define "kong.controller-container-new" -}}
 - name: ingress-controller
   securityContext:
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index 11ba29941..99ba69287 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -16,7 +16,7 @@
 # TODO 921 this needs changes to the helper template since we'll need to point to two different sets of pods.
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
-{{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
+{{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.controllerSelectorLabels" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
 # TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
@@ -26,11 +26,12 @@
 {{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
 
 # TODO 921 ditto the deploy labels todo
+{{- $_ := set $podPreRender "component" "controller" -}}
 {{- $_ := set $podPreRender "labels" (include "kong.metaLabels" .) -}}
 {{- $_ := set $podPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "controller") -}}
+# TODO 921 copy pasta as-is, but we can probably set this to the _controller_ version
 {{- $_ := set $podPreRender "versionLabel" .Chart.AppVersion -}}
 
-# TODO 921 probably use the same pre pattern for this
 {{- $containerConfig := .Values.deployment.controller.pod.container -}}
 {{- $containerPre := dict -}}
 {{- $_ := set $containerPre "admissionWebhook" .Values.admissionWebhook -}}
diff --git a/charts/kong/templates/kong-deploy.yaml b/charts/kong/templates/kong-deploy.yaml
new file mode 100644
index 000000000..76b73ccd8
--- /dev/null
+++ b/charts/kong/templates/kong-deploy.yaml
@@ -0,0 +1,68 @@
+{{- if .Values.deployment.kong.enabled }}
+{{- $deployConfig := dict -}}
+{{- $deployConfig = (mustMerge $deployConfig .Values.deployment.kong) -}}
+{{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
+{{- $podPreRender := dict -}}
+
+# TODO 921 review how these names end up in practice with a "kong" release. we've historically
+# suggested a lot of example configurations that resulted in various "kong-kong" resources which,
+# while harmless, look a bit silly
+{{- $_ := set $deployPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "proxy") -}}
+{{- $_ := set $deployPreRender "namespace" (include "kong.namespace" .) -}}
+# TODO 921 combine with .deployment.kong.labels.
+# unsure how the types work out here--kong.metaLabels produces YAML output, and I'm not sure if we can
+# just concat them as strings or what. can maybe ignore .deployment.kong.labels--it wasn't
+# included in the past and we instead set extraLabels on EVERYTHING. that's apparently sufficient for
+# user needs (there's been no request to provide separate per-resource labels here), but incongruous
+# with how we handle annotations
+{{- $_ := set $deployPreRender "labels" (include "kong.metaLabels" .) -}}
+# TODO 921 this needs changes to the helper template since we'll need to point to two different sets of pods.
+# in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
+# need to pass a suffix to the helper
+{{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
+{{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
+# TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
+# pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
+# unnecessary volumes doesn't really hurt us
+{{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
+{{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
+
+# TODO 921 ditto the deploy labels todo
+{{- $_ := set $podPreRender "component" "proxy" -}}
+{{- $_ := set $podPreRender "labels" (include "kong.metaLabels" .) -}}
+{{- $_ := set $podPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "proxy") -}}
+{{- $_ := set $podPreRender "versionLabel" .Chart.AppVersion -}}
+
+# TODO 921 rename once the old block is gone
+{{- $containerConfig := .Values.deployment.kong.pod.container -}}
+{{- $containerPre := dict -}}
+{{- $serviceConfig := dict -}}
+{{- $_ := set $containerPre "admissionWebhook" .Values.admissionWebhook -}}
+{{- $_ := set $containerPre "enterprise" .Values.enterprise -}}
+
+{{- $_ := set $serviceConfig "admin" .Values.admin -}}
+{{- $_ := set $serviceConfig "proxy" .Values.proxy -}}
+{{- $_ := set $serviceConfig "udpProxy" .Values.udpProxy -}}
+{{- $_ := set $serviceConfig "status" .Values.status -}}
+{{- $_ := set $serviceConfig "cluster" .Values.cluster -}}
+{{- $_ := set $serviceConfig "manager" .Values.manager -}}
+{{- $_ := set $serviceConfig "portal" .Values.portal -}}
+{{- $_ := set $serviceConfig "portalapi" .Values.portalapi -}}
+{{- $_ := set $serviceConfig "clustertelemetry" .Values.clustertelemetry -}}
+{{- $_ := set $containerPre "svc" $serviceConfig -}}
+
+{{- $_ := set $containerPre "serviceAccount" .Values.deployment.kong.pod.serviceAccount -}}
+{{- $_ := set $containerPre "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $containerPre "serviceAccountName" (include "kong.serviceAccountName" .) -}}
+{{- $_ := set $containerConfig "pre" $containerPre -}}
+
+{{- $_ := set $podPreRender "container" (include "kong.proxy-container-new" $containerConfig ) -}}
+
+{{- $preRender := dict -}}
+{{- $_ := set $preRender "deployment" $deployPreRender -}}
+{{- $_ := set $preRender "pod" $podPreRender -}}
+{{- $_ := set $deployConfig "pre" $preRender -}} # TODO 921 maybe think of a better key name
+
+{{- template "kong.deployment" $deployConfig -}}
+{{- end -}}
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index c63ce3715..48c3f8b18 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -173,6 +173,127 @@ deployment:
           #  memory: 2G
 
   kong:
+    enabled: true
+    labels: {}
+    annotations: {} # TODO 921 add any defaults
+    # TODO 921 the equivalent root setting has "default" values: https://github.com/Kong/charts/blob/kong-2.30.0/charts/kong/values.yaml#L887-L904
+    # These are no-oped with an `enabled=false` key, but are somewhat
+    # misleading in that we didn't really design them for any particular
+    # purpose--they're just values to show the structure. Given how values.yaml
+    # defaults work IDK if maybe we'd consider making the _actual_ values.yaml
+    # base not set defaults in these cases, and then having a dedicated example
+    # values.yaml that just shows structure.
+    autoscaling: {}
+    # TODO 921 parity, though it really makes _no_ sense for a daemonset
+    # controller. that said, we may be able to use the same templates for both
+    # Deployments' basic Deployment-y things and get it for free.
+    daemonset: false
+    updateStrategy: {}
+      # type: RollingUpdate
+      # rollingUpdate:
+      #   maxSurge: "100%"
+      #   maxUnavailable: "0%"
+    sidecarContainers: []
+    userDefinedVolumes: []
+    initContainers: []
+    pod:
+      annotations: {} # .podAnnotations, for the controller
+      # TODO 921 this should probably go elsewhere
+      serviceAccount:
+        create: true
+        # TODO 921 we arguably don't need this with a split controller
+        # Deployment, since we use it to hide the token from Kong and its
+        # arbitrary plugin codeTODO 921 we arguably don't need this with a
+        # split controller Deployment, since we use it to hide the token from
+        # Kong and its arbitrary plugin code.
+        # TODO 921 probably ignore the above, I think we still need it for the Kuma case
+        automountServiceAccountToken: false
+      hostNetwork: false
+      # Affinity for pod assignment
+      # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+      affinity: {}
+      # Topology spread constraints for pod assignment (requires Kubernetes >= 1.19)
+      # Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/
+      topologySpreadConstraints: []
+      # Sets the termination grace period for pods spawned by the Kubernetes Deployment.
+      # Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution
+      terminationGracePeriodSeconds: 30
+      # Tolerations for pod assignment
+      # Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+      tolerations: []
+      # Node labels for pod assignment
+      # Ref: https://kubernetes.io/docs/user-guide/node-selection/
+      nodeSelector: {}
+      securityContext: {}
+      container:
+        args: []
+        env:
+          database: "off"
+          router_flavor: "traditional"
+          nginx_worker_processes: "2"
+          proxy_access_log: /dev/stdout
+          admin_access_log: /dev/stdout
+          admin_gui_access_log: /dev/stdout
+          portal_api_access_log: /dev/stdout
+          proxy_error_log: /dev/stderr
+          admin_error_log: /dev/stderr
+          admin_gui_error_log: /dev/stderr
+          portal_api_error_log: /dev/stderr
+          prefix: /kong_prefix/
+        image:
+          repository: kong
+          tag: "3.4"
+          # Kong Enterprise
+          # repository: kong/kong-gateway
+          # tag: "3.4"
+          # Optionally set a semantic version for version-gated features. This can normally
+          # be left unset. You only need to set this if your tag is not a semver string,
+          # such as when you are using a "next" tag. Set this to the effective semantic
+          # version of your tag: for example if using a "next" image for an unreleased 3.1.0
+          # version, set this to "3.1.0".
+          effectiveSemver: ""
+        readinessProbe:
+          httpGet:
+            path: "/status/ready"
+            port: status
+            scheme: HTTP
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+          failureThreshold: 3
+        livenessProbe:
+          httpGet:
+            path: "/status"
+            port: status
+            scheme: HTTP
+          initialDelaySeconds: 5
+          timeoutSeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+          failureThreshold: 3
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          runAsUser: 1000
+          runAsNonRoot: true
+          seccompProfile:
+            type: RuntimeDefault
+          capabilities:
+            drop:
+            - ALL
+        userDefinedVolumeMounts: []
+        # If you want to specify resources, uncomment the following
+        # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+        resources: {}
+          # limits:
+          #  cpu: 1
+          #  memory: 2G
+          # requests:
+          #  cpu: 1
+          #  memory: 2G
+
+  oldkong:
     # Enable or disable Kong itself
     # Setting this to false with ingressController.enabled=true will create a
     # controller-only release.

From 01de9e161981188e0003aae4bee99e0b725be52f Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Fri, 10 Nov 2023 15:10:51 -0800
Subject: [PATCH 20/24] wip: buncha env and volume stuff; broken template

---
 charts/kong/templates/_helpers.tpl            | 365 +-----------------
 charts/kong/templates/containers.yaml         | 227 +++++++++++
 charts/kong/templates/controller-deploy.yaml  |   5 +-
 .../kong/templates/deployment-controller.yaml |  10 +-
 charts/kong/templates/env.yaml                | 280 ++++++++++++++
 charts/kong/templates/kong-deploy.yaml        |  80 +++-
 charts/kong/values.yaml                       |  24 +-
 7 files changed, 611 insertions(+), 380 deletions(-)
 create mode 100644 charts/kong/templates/containers.yaml
 create mode 100644 charts/kong/templates/env.yaml

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index 56cbb9789..822fb3e70 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -49,7 +49,7 @@ app.kubernetes.io/instance: "{{ .Release.Name }}"
 {{- end -}}
 
 {{- define "kong.postgresql.fullname" -}}
-{{- $name := default "postgresql" .Values.postgresql.nameOverride -}}
+{{- $name := default "postgresql" .postgresql.nameOverride -}}
 {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
 {{- end -}}
 
@@ -466,104 +466,6 @@ The name of the Service which will be used by the controller to update the Ingre
 {{- (printf "%s/%s" ( include "kong.namespace" . ) ( default ( printf "%s-proxy" (include "kong.fullname" . )) $proxyOverride )) -}}
 {{- end -}}
 
-# TODO 921 everything here would need to become relative. some of it relies on
-# external values though, e.g. .Values.ingressController.ingressClass and
-# .Values.ingressController.admissionWebhook.address presumably remain where
-# they are.
-{{- define "kong.ingressController.env" -}}
-{{/*
-    ====== AUTO-GENERATED ENVIRONMENT VARIABLES ======
-*/}}
-
-
-{{- $autoEnv := dict -}}
-  {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY" true -}}
-  {{- $_ := set $autoEnv "CONTROLLER_PUBLISH_SERVICE" ( include "kong.controller-publish-service" . ) -}}
-  {{- $_ := set $autoEnv "CONTROLLER_INGRESS_CLASS" .Values.ingressController.ingressClass -}}
-  {{- $_ := set $autoEnv "CONTROLLER_ELECTION_ID" (printf "kong-ingress-controller-leader-%s" .Values.ingressController.ingressClass) -}}
-
-  {{- if .Values.ingressController.admissionWebhook.enabled }}
-    {{- $address := (default "0.0.0.0" .Values.ingressController.admissionWebhook.address) -}}
-    {{- $_ := set $autoEnv "CONTROLLER_ADMISSION_WEBHOOK_LISTEN" (printf "%s:%d" $address (int64 .Values.ingressController.admissionWebhook.port)) -}}
-  {{- end }}
-  {{- if (not (eq (len .Values.ingressController.watchNamespaces) 0)) }}
-    {{- $_ := set $autoEnv "CONTROLLER_WATCH_NAMESPACE" (.Values.ingressController.watchNamespaces | join ",") -}}
-  {{- end }}
-
-{{/*
-    ====== ADMIN API CONFIGURATION ======
-*/}}
-
-  {{- if .Values.ingressController.gatewayDiscovery.enabled -}}
-    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_SVC" (include "kong.adminSvc" . ) -}}
-  {{- else -}}
-    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_URL" (include "kong.adminLocalURL" .) -}}
-  {{- end -}}
-
-  {{- if .Values.ingressController.adminApi.tls.client.enabled }}
-    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_CLIENT_CERT_FILE" "/etc/secrets/admin-api-cert/tls.crt" -}}
-    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_CLIENT_KEY_FILE" "/etc/secrets/admin-api-cert/tls.key" -}}
-  {{- end }}
-
-{{/*
-    ====== KONNECT ENVIRONMENT VARIABLES ======
-*/}}
-
-{{- if .Values.ingressController.konnect.enabled }}
-  {{- if (semverCompare "< 2.9.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
-  {{- fail (printf "Konnect sync is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
-  {{- end }}
-
-  {{- if not .Values.ingressController.gatewayDiscovery.enabled }}
-  {{- fail "ingressController.gatewayDiscovery.enabled has to be true when ingressController.konnect.enabled"}}
-  {{- end }}
-
-  {{- $konnect := .Values.ingressController.konnect -}}
-  {{- $_ := required "ingressController.konnect.runtimeGroupID is required when ingressController.konnect.enabled" $konnect.runtimeGroupID -}}
-
-  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_SYNC_ENABLED" true -}}
-  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_RUNTIME_GROUP_ID" $konnect.runtimeGroupID -}}
-  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_ADDRESS" (printf "https://%s" .Values.ingressController.konnect.apiHostname) -}}
-
-  {{- $tlsCert := include "secretkeyref" (dict "name" $konnect.tlsClientCertSecretName "key" "tls.crt") -}}
-  {{- $tlsKey := include "secretkeyref" (dict "name" $konnect.tlsClientCertSecretName "key" "tls.key") -}}
-  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_TLS_CLIENT_CERT" $tlsCert -}}
-  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_TLS_CLIENT_KEY" $tlsKey -}}
-
-  {{- if $konnect.license.enabled }}
-  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_LICENSING_ENABLED" true -}}
-  {{- end }}
-{{- end }}
-
-{{/*
-    ====== USER-SET ENVIRONMENT VARIABLES ======
-*/}}
-
-{{- $userEnv := dict -}}
-{{- range $key, $val := .Values.deployment.controller.pod.container.env }}
-  {{- $upper := upper $key -}}
-  {{- $var := printf "CONTROLLER_%s" $upper -}}
-  {{- $_ := set $userEnv $var $val -}}
-{{- end -}}
-
-{{/*
-    ====== CUSTOM-SET INGRESS CONTROLLER ENVIRONMENT VARIABLES ======
-*/}}
-
-{{- $customIngressEnv := dict -}}
-{{- range $key, $val := .Values.ingressController.customEnv }}
-  {{- $upper := upper $key -}}
-  {{- $_ := set $customIngressEnv $upper $val -}}
-{{- end -}}
-
-{{/*
-      ====== MERGE AND RENDER ENV BLOCK ======
-*/}}
-
-{{- $completeEnv := mergeOverwrite $autoEnv $userEnv $customIngressEnv -}}
-{{- template "kong.renderEnv" $completeEnv -}}
-
-{{- end -}}
 
 {{- define "kong.userDefinedVolumes" -}}
 {{- if .Values.deployment.userDefinedVolumes }}
@@ -816,12 +718,14 @@ The name of the Service which will be used by the controller to update the Ingre
 
 {{- end -}}
 
+{{/* TODO 921 this gets used for any Kong env. the migrations and init paths here don't yet
+     build a dict, and so won't work without the removed .Values */}}
 {{- define "kong.plugins" -}}
 {{ $myList := list "bundled" }}
-{{- range .Values.plugins.configMaps -}}
+{{- range .pre.plugins.configMaps -}}
 {{- $myList = append $myList .pluginName -}}
 {{- end -}}
-{{- range .Values.plugins.secrets -}}
+{{- range .pre.plugins.secrets -}}
   {{ $myList = append $myList .pluginName -}}
 {{- end }}
 {{- $myList | uniq | join "," -}}
@@ -929,236 +833,6 @@ The name of the Service which will be used by the controller to update the Ingre
   {{- include "controller.adminApiCertVolumeMount" . | nindent 2 }}
 {{- end -}}
 
-{{- define "kong.proxy-container-new" -}}
-- name: "proxy"
-  image: {{ include "kong.getRepoTag" .image }}
-  imagePullPolicy: {{ .image.pullPolicy }}
-  securityContext:
-  {{ toYaml .securityContext | nindent 4 }}
-{{/*   TODO 921 this helper still pulls from the old .Values.env. We need to pass in the deployment.kong.pod.container.env
-  env:
-  {{- (include "kong.no_daemon_env" .) | nindent 2 }}
-*/}}
-  lifecycle:
-    {{- toYaml .lifecycle | nindent 4 }}
-  ports:
-  {{- if (and .pre.svc.admin.http.enabled .pre.svc.admin.enabled) }}
-  - name: admin
-    containerPort: {{ .pre.svc.admin.http.containerPort }}
-    {{- if .pre.svc.admin.http.hostPort }}
-    hostPort: {{ .pre.svc.admin.http.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.admin.tls.enabled .pre.svc.admin.enabled) }}
-  - name: admin-tls
-    containerPort: {{ .pre.svc.admin.tls.containerPort }}
-    {{- if .pre.svc.admin.tls.hostPort }}
-    hostPort: {{ .pre.svc.admin.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.proxy.http.enabled .pre.svc.proxy.enabled) }}
-  - name: proxy
-    containerPort: {{ .pre.svc.proxy.http.containerPort }}
-    {{- if .pre.svc.proxy.http.hostPort }}
-    hostPort: {{ .pre.svc.proxy.http.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.proxy.tls.enabled .pre.svc.proxy.enabled)}}
-  - name: proxy-tls
-    containerPort: {{ .pre.svc.proxy.tls.containerPort }}
-    {{- if .pre.svc.proxy.tls.hostPort }}
-    hostPort: {{ .pre.svc.proxy.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- range .pre.svc.proxy.stream }}
-  - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
-    containerPort: {{ .containerPort }}
-    {{- if .hostPort }}
-    hostPort: {{ .hostPort }}
-    {{- end}}
-    protocol: {{ .protocol }}
-  {{- end }}
-  {{- range .pre.svc.udpProxy.stream }}
-  - name: streamudp-{{ .containerPort }}
-    containerPort: {{ .containerPort }}
-    {{- if .hostPort }}
-    hostPort: {{ .hostPort }}
-    {{- end}}
-    protocol: {{ .protocol }}
-  {{- end }}
-  {{- if (and .pre.svc.status.http.enabled .pre.svc.status.enabled)}}
-  - name: status
-    containerPort: {{ .pre.svc.status.http.containerPort }}
-    {{- if .pre.svc.status.http.hostPort }}
-    hostPort: {{ .pre.svc.status.http.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.status.tls.enabled .pre.svc.status.enabled) }}
-  - name: status-tls
-    containerPort: {{ .pre.svc.status.tls.containerPort }}
-    {{- if .pre.svc.status.tls.hostPort }}
-    hostPort: {{ .pre.svc.status.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.cluster.tls.enabled .pre.svc.cluster.enabled) }}
-  - name: cluster-tls
-    containerPort: {{ .pre.svc.cluster.tls.containerPort }}
-    {{- if .pre.svc.cluster.tls.hostPort }}
-    hostPort: {{ .pre.svc.cluster.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if .pre.enterprise.enabled }}
-  {{- if (and .pre.svc.manager.http.enabled .pre.svc.manager.enabled) }}
-  - name: manager
-    containerPort: {{ .pre.svc.manager.http.containerPort }}
-    {{- if .pre.svc.manager.http.hostPort }}
-    hostPort: {{ .pre.svc.manager.http.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.manager.tls.enabled .pre.svc.manager.enabled) }}
-  - name: manager-tls
-    containerPort: {{ .pre.svc.manager.tls.containerPort }}
-    {{- if .pre.svc.manager.tls.hostPort }}
-    hostPort: {{ .pre.svc.manager.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.portal.http.enabled .pre.svc.portal.enabled) }}
-  - name: portal
-    containerPort: {{ .pre.svc.portal.http.containerPort }}
-    {{- if .pre.svc.portal.http.hostPort }}
-    hostPort: {{ .pre.svc.portal.http.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.portal.tls.enabled .pre.svc.portal.enabled) }}
-  - name: portal-tls
-    containerPort: {{ .pre.svc.portal.tls.containerPort }}
-    {{- if .pre.svc.portal.tls.hostPort }}
-    hostPort: {{ .pre.svc.portal.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.portalapi.http.enabled .pre.svc.portalapi.enabled) }}
-  - name: portalapi
-    containerPort: {{ .pre.svc.portalapi.http.containerPort }}
-    {{- if .pre.svc.portalapi.http.hostPort }}
-    hostPort: {{ .pre.svc.portalapi.http.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.portalapi.tls.enabled .pre.svc.portalapi.enabled) }}
-  - name: portalapi-tls
-    containerPort: {{ .pre.svc.portalapi.tls.containerPort }}
-    {{- if .pre.svc.portalapi.tls.hostPort }}
-    hostPort: {{ .pre.svc.portalapi.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- if (and .pre.svc.clustertelemetry.tls.enabled .pre.svc.clustertelemetry.enabled) }}
-  - name: clustert-tls
-    containerPort: {{ .pre.svc.clustertelemetry.tls.containerPort }}
-    {{- if .pre.svc.clustertelemetry.tls.hostPort }}
-    hostPort: {{ .pre.svc.clustertelemetry.tls.hostPort }}
-    {{- end}}
-    protocol: TCP
-  {{- end }}
-  {{- end }}
-{{/* TODO 921 whatever we're doing for volume mounts
-  volumeMounts:
-  {{- include "kong.volumeMounts" . | nindent 4 }}
-  {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 4 }}
-*/}}
-  readinessProbe:
-{{ toYaml .readinessProbe | indent 4 }}
-{{/* TODO 921 this requires controller-related keys we aren't passing in yet
-{{ include "kong.proxy.compatibleReadiness" . | indent 4 }}
-*/}}
-  livenessProbe:
-{{ toYaml .livenessProbe | indent 4 }}
-  {{- if .startupProbe }}
-  startupProbe:
-{{ toYaml .startupProbe | indent 4 }}
-  {{- end }}
-  resources:
-{{ toYaml .resources | indent 4 }}
-{{- end -}}
-
-{{- define "kong.controller-container-new" -}}
-- name: ingress-controller
-  securityContext:
-{{ toYaml .securityContext | nindent 4 }}
-  args:
-  {{ if .args}}
-  {{- range $val := .args }}
-  - {{ $val }}
-  {{- end }}
-  {{- end }}
-  ports:
-  {{- if .pre.admissionWebhook.enabled }}
-  - name: webhook
-    containerPort: {{ .pre.admissionWebhook.port }}
-    protocol: TCP
-  {{- end }}
-  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .image)) -}}
-  - name: cmetrics
-    containerPort: 10255
-    protocol: TCP
-  {{- end }}
-  env:
-  - name: POD_NAME
-    valueFrom:
-      fieldRef:
-        apiVersion: v1
-        fieldPath: metadata.name
-  - name: POD_NAMESPACE
-    valueFrom:
-      fieldRef:
-        apiVersion: v1
-        fieldPath: metadata.namespace
-# TODO 921 this is not currently designed to be at all relative and relies on
-# external sections. the template invoker renders this from the root and passes
-# it in as .env
-  env:
-{{- .pre.env  | indent 4 }}
-  image: {{ include "kong.getRepoTag" .image }}
-  imagePullPolicy: {{ .image.pullPolicy }}
-{{/* disableReadiness is a hidden setting to drop this block entirely for use with a debugger
-     Helm value interpretation doesn't let you replace the default HTTP checks with any other
-     check type, and all HTTP checks freeze when a debugger pauses operation.
-     Setting disableReadiness to ANY value disables the probes.
-*/}}
-{{- if (not (hasKey .pre "disableProbes")) }}
-  readinessProbe:
-{{ toYaml .readinessProbe | indent 4 }}
-  livenessProbe:
-{{ toYaml .livenessProbe | indent 4 }}
-{{- end }}
-  resources:
-{{ toYaml .resources | indent 4 }}
-  volumeMounts:
-{{- if .pre.admissionWebhook.enabled }}
-  - name: webhook-cert
-    mountPath: /admission-webhook
-    readOnly: true
-{{- end }}
-{{- if (and (not .pre.serviceAccount.automountServiceAccountToken) (or .pre.serviceAccount.create .pre.serviceAccount.name)) }}
-  - name: {{ .pre.serviceAccountTokenName }}
-    mountPath: /var/run/secrets/kubernetes.io/serviceaccount
-    readOnly: true
-{{- end }}
-  {{- include "kong.userDefinedVolumeMounts" . | nindent 2 }}
-  # TODO 921 relies on some content under .ingressController
-  {{- include "controller.adminApiCertVolumeMount" .pre | nindent 2 }}
-{{- end -}}
 
 {{- define "secretkeyref" -}}
 valueFrom:
@@ -1414,35 +1088,6 @@ the template that it itself is using form the above sections.
 
 {{- end -}}
 
-{{/*
-Given a dictionary of variable=value pairs, render a container env block.
-Environment variables are sorted alphabetically
-*/}}
-{{- define "kong.renderEnv" -}}
-
-{{- $dict := . -}}
-
-{{- range keys . | sortAlpha }}
-{{- $val := pluck . $dict | first -}}
-{{- $valueType := printf "%T" $val -}}
-{{ if eq $valueType "map[string]interface {}" }}
-- name: {{ . }}
-{{ toYaml $val | indent 2 -}}
-{{- else if eq $valueType "string" }}
-{{- if regexMatch "valueFrom" $val }}
-- name: {{ . }}
-{{ $val | indent 2 }}
-{{- else }}
-- name: {{ . }}
-  value: {{ $val | quote }}
-{{- end }}
-{{- else }}
-- name: {{ . }}
-  value: {{ $val | quote }}
-{{- end }}
-{{- end -}}
-
-{{- end -}}
 
 {{- define "kong.wait-for-postgres" -}}
 - name: wait-for-postgres
diff --git a/charts/kong/templates/containers.yaml b/charts/kong/templates/containers.yaml
new file mode 100644
index 000000000..2458a27eb
--- /dev/null
+++ b/charts/kong/templates/containers.yaml
@@ -0,0 +1,227 @@
+{{- define "kong.proxy-container-new" -}}
+- name: "proxy"
+  image: {{ include "kong.getRepoTag" .image }}
+  imagePullPolicy: {{ .image.pullPolicy }}
+  securityContext:
+  {{ toYaml .securityContext | nindent 4 }}
+  env:
+  {{- (include "kong.env.proxy.noDaemon" .) | nindent 2 }}
+  lifecycle:
+    {{- toYaml .lifecycle | nindent 4 }}
+  ports:
+  {{- if (and .pre.svc.admin.http.enabled .pre.svc.admin.enabled) }}
+  - name: admin
+    containerPort: {{ .pre.svc.admin.http.containerPort }}
+    {{- if .pre.svc.admin.http.hostPort }}
+    hostPort: {{ .pre.svc.admin.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.admin.tls.enabled .pre.svc.admin.enabled) }}
+  - name: admin-tls
+    containerPort: {{ .pre.svc.admin.tls.containerPort }}
+    {{- if .pre.svc.admin.tls.hostPort }}
+    hostPort: {{ .pre.svc.admin.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.proxy.http.enabled .pre.svc.proxy.enabled) }}
+  - name: proxy
+    containerPort: {{ .pre.svc.proxy.http.containerPort }}
+    {{- if .pre.svc.proxy.http.hostPort }}
+    hostPort: {{ .pre.svc.proxy.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.proxy.tls.enabled .pre.svc.proxy.enabled)}}
+  - name: proxy-tls
+    containerPort: {{ .pre.svc.proxy.tls.containerPort }}
+    {{- if .pre.svc.proxy.tls.hostPort }}
+    hostPort: {{ .pre.svc.proxy.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- range .pre.svc.proxy.stream }}
+  - name: stream{{ if (eq (default "TCP" .protocol) "UDP") }}udp{{ end }}-{{ .containerPort }}
+    containerPort: {{ .containerPort }}
+    {{- if .hostPort }}
+    hostPort: {{ .hostPort }}
+    {{- end}}
+    protocol: {{ .protocol }}
+  {{- end }}
+  {{- range .pre.svc.udpProxy.stream }}
+  - name: streamudp-{{ .containerPort }}
+    containerPort: {{ .containerPort }}
+    {{- if .hostPort }}
+    hostPort: {{ .hostPort }}
+    {{- end}}
+    protocol: {{ .protocol }}
+  {{- end }}
+  {{- if (and .pre.svc.status.http.enabled .pre.svc.status.enabled)}}
+  - name: status
+    containerPort: {{ .pre.svc.status.http.containerPort }}
+    {{- if .pre.svc.status.http.hostPort }}
+    hostPort: {{ .pre.svc.status.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.status.tls.enabled .pre.svc.status.enabled) }}
+  - name: status-tls
+    containerPort: {{ .pre.svc.status.tls.containerPort }}
+    {{- if .pre.svc.status.tls.hostPort }}
+    hostPort: {{ .pre.svc.status.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.cluster.tls.enabled .pre.svc.cluster.enabled) }}
+  - name: cluster-tls
+    containerPort: {{ .pre.svc.cluster.tls.containerPort }}
+    {{- if .pre.svc.cluster.tls.hostPort }}
+    hostPort: {{ .pre.svc.cluster.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if .pre.enterprise.enabled }}
+  {{- if (and .pre.svc.manager.http.enabled .pre.svc.manager.enabled) }}
+  - name: manager
+    containerPort: {{ .pre.svc.manager.http.containerPort }}
+    {{- if .pre.svc.manager.http.hostPort }}
+    hostPort: {{ .pre.svc.manager.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.manager.tls.enabled .pre.svc.manager.enabled) }}
+  - name: manager-tls
+    containerPort: {{ .pre.svc.manager.tls.containerPort }}
+    {{- if .pre.svc.manager.tls.hostPort }}
+    hostPort: {{ .pre.svc.manager.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portal.http.enabled .pre.svc.portal.enabled) }}
+  - name: portal
+    containerPort: {{ .pre.svc.portal.http.containerPort }}
+    {{- if .pre.svc.portal.http.hostPort }}
+    hostPort: {{ .pre.svc.portal.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portal.tls.enabled .pre.svc.portal.enabled) }}
+  - name: portal-tls
+    containerPort: {{ .pre.svc.portal.tls.containerPort }}
+    {{- if .pre.svc.portal.tls.hostPort }}
+    hostPort: {{ .pre.svc.portal.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portalapi.http.enabled .pre.svc.portalapi.enabled) }}
+  - name: portalapi
+    containerPort: {{ .pre.svc.portalapi.http.containerPort }}
+    {{- if .pre.svc.portalapi.http.hostPort }}
+    hostPort: {{ .pre.svc.portalapi.http.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.portalapi.tls.enabled .pre.svc.portalapi.enabled) }}
+  - name: portalapi-tls
+    containerPort: {{ .pre.svc.portalapi.tls.containerPort }}
+    {{- if .pre.svc.portalapi.tls.hostPort }}
+    hostPort: {{ .pre.svc.portalapi.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- if (and .pre.svc.clustertelemetry.tls.enabled .pre.svc.clustertelemetry.enabled) }}
+  - name: clustert-tls
+    containerPort: {{ .pre.svc.clustertelemetry.tls.containerPort }}
+    {{- if .pre.svc.clustertelemetry.tls.hostPort }}
+    hostPort: {{ .pre.svc.clustertelemetry.tls.hostPort }}
+    {{- end}}
+    protocol: TCP
+  {{- end }}
+  {{- end }}
+  volumeMounts:
+  {{- .pre.volumeMounts | nindent 4 }}
+  {{- .userDefinedVolumeMounts | toYaml | nindent 4 }}
+  readinessProbe:
+{{ toYaml .readinessProbe | indent 4 }}
+{{/* TODO 921 this requires controller-related keys we aren't passing in yet
+{{ include "kong.proxy.compatibleReadiness" . | indent 4 }}
+*/}}
+  livenessProbe:
+{{ toYaml .livenessProbe | indent 4 }}
+  {{- if .startupProbe }}
+  startupProbe:
+{{ toYaml .startupProbe | indent 4 }}
+  {{- end }}
+  resources:
+{{ toYaml .resources | indent 4 }}
+{{- end -}}
+
+{{- define "kong.controller-container-new" -}}
+- name: ingress-controller
+  securityContext:
+{{ toYaml .securityContext | nindent 4 }}
+  args:
+  {{ if .args}}
+  {{- range $val := .args }}
+  - {{ $val }}
+  {{- end }}
+  {{- end }}
+  ports:
+  {{- if .pre.admissionWebhook.enabled }}
+  - name: webhook
+    containerPort: {{ .pre.admissionWebhook.port }}
+    protocol: TCP
+  {{- end }}
+  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .image)) -}}
+  - name: cmetrics
+    containerPort: 10255
+    protocol: TCP
+  {{- end }}
+  env:
+  - name: POD_NAME
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.name
+  - name: POD_NAMESPACE
+    valueFrom:
+      fieldRef:
+        apiVersion: v1
+        fieldPath: metadata.namespace
+# TODO 921 this is not currently designed to be at all relative and relies on
+# external sections. the template invoker renders this from the root and passes
+# it in as .env
+  env:
+{{- .pre.env  | indent 4 }}
+  image: {{ include "kong.getRepoTag" .image }}
+  imagePullPolicy: {{ .image.pullPolicy }}
+{{/* disableReadiness is a hidden setting to drop this block entirely for use with a debugger
+     Helm value interpretation doesn't let you replace the default HTTP checks with any other
+     check type, and all HTTP checks freeze when a debugger pauses operation.
+     Setting disableReadiness to ANY value disables the probes.
+*/}}
+{{- if (not (hasKey .pre "disableProbes")) }}
+  readinessProbe:
+{{ toYaml .readinessProbe | indent 4 }}
+  livenessProbe:
+{{ toYaml .livenessProbe | indent 4 }}
+{{- end }}
+  resources:
+{{ toYaml .resources | indent 4 }}
+  volumeMounts:
+{{- if .pre.admissionWebhook.enabled }}
+  - name: webhook-cert
+    mountPath: /admission-webhook
+    readOnly: true
+{{- end }}
+{{- if (and (not .pre.serviceAccount.automountServiceAccountToken) (or .pre.serviceAccount.create .pre.serviceAccount.name)) }}
+  - name: {{ .pre.serviceAccountTokenName }}
+    mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+    readOnly: true
+{{- end }}
+  {{- include "kong.userDefinedVolumeMounts" . | nindent 2 }}
+  # TODO 921 relies on some content under .ingressController
+  {{- include "controller.adminApiCertVolumeMount" .pre | nindent 2 }}
+{{- end -}}
+
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index 99ba69287..c0fbf32c2 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -23,10 +23,12 @@
 # pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
 # unnecessary volumes doesn't really hurt us
 {{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
+# TODO 921 this is not pulling from the new location
 {{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "component" "controller" -}}
+{{- $_ := set $podPreRender "app" (include "kong.fullname" .) -}}
 {{- $_ := set $podPreRender "labels" (include "kong.metaLabels" .) -}}
 {{- $_ := set $podPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "controller") -}}
 # TODO 921 copy pasta as-is, but we can probably set this to the _controller_ version
@@ -35,7 +37,8 @@
 {{- $containerConfig := .Values.deployment.controller.pod.container -}}
 {{- $containerPre := dict -}}
 {{- $_ := set $containerPre "admissionWebhook" .Values.admissionWebhook -}}
-{{- $_ := set $containerPre "env" (include "kong.ingressController.env" .) -}}
+# TODO 921 still global scoped
+{{- $_ := set $containerPre "env" (include "kong.env.controller" .) -}}
 {{- $_ := set $containerPre "disableProbes" .Values.ingressController.disableProbes -}}
 {{- $_ := set $containerPre "adminApi" .Values.ingressController.adminApi -}}
 {{- $_ := set $containerPre "serviceAccount" .Values.deployment.controller.pod.serviceAccount -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 31603cb10..24b8ee820 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -66,8 +66,8 @@ spec:
         {{- end }}
       labels:
         {{- .pre.pod.labels | nindent 8 }}
-        app.kubernetes.io/component: app
-        app: {{ .pre.pod.name }}
+        app.kubernetes.io/component: {{ .pre.pod.component }}
+        app: {{ .pre.pod.app }}
         version: {{ .pre.pod.versionLabel | quote }}
         {{- if .pod.labels }}
         {{ include "kong.renderTpl" (dict "value" .pod.labels "context" $) | nindent 8 }}
@@ -138,7 +138,11 @@ spec:
       dnsConfig:
 {{ toYaml .pod.dnsConfig | indent 8 }}
       {{- end }}
-      containers: {{ .pre.pod.container | nindent 8 }}
+      containers:
+{{ .pre.pod.container | nindent 8 }}
+{{- if .pod.sidecarContainers }}
+{{- toYaml .pod.sidecarContainers | nindent 8 }}
+{{- end }}
     {{- if .pod.affinity }}
       affinity:
 {{ toYaml .pod.affinity | indent 8 }}
diff --git a/charts/kong/templates/env.yaml b/charts/kong/templates/env.yaml
new file mode 100644
index 000000000..b5ce7f0fb
--- /dev/null
+++ b/charts/kong/templates/env.yaml
@@ -0,0 +1,280 @@
+{{/*
+Given a dictionary of variable=value pairs, render a container env block.
+Environment variables are sorted alphabetically
+*/}}
+{{- define "kong.renderEnv" -}}
+
+{{- $dict := . -}}
+
+{{- range keys . | sortAlpha }}
+{{- $val := pluck . $dict | first -}}
+{{- $valueType := printf "%T" $val -}}
+{{ if eq $valueType "map[string]interface {}" }}
+- name: {{ . }}
+{{ toYaml $val | indent 2 -}}
+{{- else if eq $valueType "string" }}
+{{- if regexMatch "valueFrom" $val }}
+- name: {{ . }}
+{{ $val | indent 2 }}
+{{- else }}
+- name: {{ . }}
+  value: {{ $val | quote }}
+{{- end }}
+{{- else }}
+- name: {{ . }}
+  value: {{ $val | quote }}
+{{- end }}
+{{- end -}}
+
+{{- end -}}
+
+
+{{- define "kong.env.proxy.noDaemon" -}}
+{{- template "kong.env.proxy.basic" . }}
+- name: KONG_NGINX_DAEMON
+  value: "off"
+{{- end -}}
+
+
+{{/*
+The environment values passed to Kong; this should come after all
+the template that it itself is using form the above sections.
+*/}}
+{{- define "kong.env.proxy.basic" -}}
+{{/*
+    ====== AUTO-GENERATED ENVIRONMENT VARIABLES ======
+*/}}
+{{- $autoEnv := dict -}}
+
+{{- $_ := set $autoEnv "KONG_LUA_PACKAGE_PATH" "/opt/?.lua;/opt/?/init.lua;;" -}}
+
+{{- $_ := set $autoEnv "KONG_PROXY_ACCESS_LOG" "/dev/stdout" -}}
+{{- $_ := set $autoEnv "KONG_PROXY_STREAM_ACCESS_LOG" "/dev/stdout basic" -}}
+{{- $_ := set $autoEnv "KONG_ADMIN_ACCESS_LOG" "/dev/stdout" -}}
+{{- $_ := set $autoEnv "KONG_STATUS_ACCESS_LOG" "off" -}}
+{{- $_ := set $autoEnv "KONG_PROXY_ERROR_LOG" "/dev/stderr" -}}
+{{- $_ := set $autoEnv "KONG_PROXY_STREAM_ERROR_LOG" "/dev/stderr" -}}
+{{- $_ := set $autoEnv "KONG_ADMIN_ERROR_LOG" "/dev/stderr" -}}
+{{- $_ := set $autoEnv "KONG_STATUS_ERROR_LOG" "/dev/stderr" -}}
+
+{{- if .pre.ingressControllerEnabled -}}
+  {{- $_ := set $autoEnv "KONG_KIC" "on" -}}
+{{- end -}}
+
+{{- with .pre.svc.admin -}}
+  {{- $address := "0.0.0.0" -}}
+  {{- if (not .enabled) -}}
+    {{- $address = "127.0.0.1" -}}
+  {{- end -}}
+  {{- $listenConfig := dict -}}
+  {{- $listenConfig := merge $listenConfig . -}}
+  {{- $_ := set $listenConfig "address" (default $address .address) -}}
+  {{- $_ := set $autoEnv "KONG_ADMIN_LISTEN" (include "kong.listen" $listenConfig) -}}
+
+  {{- if or .tls.client.secretName .tls.client.caBundle -}}
+    {{- $_ := set $autoEnv "KONG_NGINX_ADMIN_SSL_VERIFY_CLIENT" "on" -}}
+    {{- $_ := set $autoEnv "KONG_NGINX_ADMIN_SSL_CLIENT_CERTIFICATE" "/etc/admin-client-ca/tls.crt" -}}
+  {{- end -}}
+
+{{- end -}}
+
+{{- if and ( .pre.Capabilities.APIVersions.Has "cert-manager.io/v1" ) .pre.certificates.enabled -}}
+  {{- if (and .pre.certificates.cluster.enabled .pre.svc.cluster.enabled) -}}
+    {{- $_ := set $autoEnv "KONG_CLUSTER_MTLS" "pki" -}}
+    {{- $_ := set $autoEnv "KONG_CLUSTER_SERVER_NAME" .pre.certificates.cluster.commonName -}}
+    {{- $_ := set $autoEnv "KONG_CLUSTER_CA_CERT" "/etc/cert-manager/cluster/ca.crt" -}}
+    {{- $_ := set $autoEnv "KONG_CLUSTER_CERT" "/etc/cert-manager/cluster/tls.crt" -}}
+    {{- $_ := set $autoEnv "KONG_CLUSTER_CERT_KEY" "/etc/cert-manager/cluster/tls.key" -}}
+  {{- end -}}
+
+  {{- if .pre.certificates.proxy.enabled -}}
+    {{- $_ := set $autoEnv "KONG_SSL_CERT" "/etc/cert-manager/proxy/tls.crt" -}}
+    {{- $_ := set $autoEnv "KONG_SSL_CERT_KEY" "/etc/cert-manager/proxy/tls.key" -}}
+  {{- end -}}
+
+  {{- if .pre.certificates.admin.enabled -}}
+    {{- $_ := set $autoEnv "KONG_ADMIN_SSL_CERT" "/etc/cert-manager/admin/tls.crt" -}}
+    {{- $_ := set $autoEnv "KONG_ADMIN_SSL_CERT_KEY" "/etc/cert-manager/admin/tls.key" -}}
+    {{- if .pre.enterprise.enabled }}
+      {{- $_ := set $autoEnv "KONG_ADMIN_GUI_SSL_CERT" "/etc/cert-manager/admin/tls.crt" -}}
+      {{- $_ := set $autoEnv "KONG_ADMIN_GUI_SSL_CERT_KEY" "/etc/cert-manager/admin/tls.key" -}}
+    {{- end -}}
+  {{- end -}}
+
+  {{- if .pre.enterprise.enabled }}
+    {{- if .pre.certificates.portal.enabled -}}
+      {{- $_ := set $autoEnv "KONG_PORTAL_API_SSL_CERT" "/etc/cert-manager/portal/tls.crt" -}}
+      {{- $_ := set $autoEnv "KONG_PORTAL_API_SSL_CERT_KEY" "/etc/cert-manager/portal/tls.key" -}}
+      {{- $_ := set $autoEnv "KONG_PORTAL_GUI_SSL_CERT" "/etc/cert-manager/portal/tls.crt" -}}
+      {{- $_ := set $autoEnv "KONG_PORTAL_GUI_SSL_CERT_KEY" "/etc/cert-manager/portal/tls.key" -}}
+    {{- end -}}
+  {{- end -}}
+{{- end -}}
+
+{{- if .pre.svc.admin.ingress.enabled }}
+  {{- $_ := set $autoEnv "KONG_ADMIN_GUI_API_URL" (include "kong.ingress.serviceUrl" .pre.svc.admin.ingress) -}}
+  {{- $_ := set $autoEnv "KONG_ADMIN_API_URI" (include "kong.ingress.serviceUrl" .pre.svc.admin.ingress) -}}
+{{- end -}}
+
+{{- $_ := set $autoEnv "KONG_PROXY_LISTEN" (include "kong.listen" .pre.svc.proxy) -}}
+
+{{- $streamStrings := list -}}
+{{- if .pre.svc.proxy.enabled -}}
+  {{- $tcpStreamString := (include "kong.streamListen" .pre.svc.proxy) -}}
+  {{- if (not (eq $tcpStreamString "")) -}}
+    {{- $streamStrings = (append $streamStrings $tcpStreamString) -}}
+  {{- end -}}
+{{- end -}}
+{{- if .pre.svc.udpProxy.enabled -}}
+  {{- $udpStreamString := (include "kong.streamListen" .pre.svc.udpProxy) -}}
+  {{- if (not (eq $udpStreamString "")) -}}
+    {{- $streamStrings = (append $streamStrings $udpStreamString) -}}
+  {{- end -}}
+{{- end -}}
+{{- $streamString := $streamStrings | join ", " -}}
+{{- if (eq (len $streamString) 0)  -}}
+  {{- $streamString = "off" -}}
+{{- end -}}
+{{- $_ := set $autoEnv "KONG_STREAM_LISTEN" $streamString -}}
+
+{{- $_ := set $autoEnv "KONG_STATUS_LISTEN" (include "kong.listen" .pre.svc.status) -}}
+
+{{- if .pre.svc.proxy.enabled -}}
+  {{- $_ := set $autoEnv "KONG_PORT_MAPS" (include "kong.port_maps" .pre.svc.proxy) -}}
+{{- end -}}
+
+{{- $_ := set $autoEnv "KONG_CLUSTER_LISTEN" (include "kong.listen" .pre.svc.cluster) -}}
+
+{{- if .pre.enterprise.enabled }}
+  {{- $_ := set $autoEnv "KONG_PORTAL_API_ACCESS_LOG" "/dev/stdout" -}}
+  {{- $_ := set $autoEnv "KONG_PORTAL_GUI_ACCESS_LOG" "/dev/stdout" -}}
+  {{- $_ := set $autoEnv "KONG_ADMIN_GUI_ACCESS_LOG" "/dev/stdout" -}}
+  {{- $_ := set $autoEnv "KONG_PORTAL_API_ERROR_LOG" "/dev/stderr" -}}
+  {{- $_ := set $autoEnv "KONG_PORTAL_GUI_ERROR_LOG" "/dev/stderr" -}}
+  {{- $_ := set $autoEnv "KONG_ADMIN_GUI_ERROR_LOG" "/dev/stderr" -}}
+
+  {{- $_ := set $autoEnv "KONG_ADMIN_GUI_LISTEN" (include "kong.listen" .pre.svc.manager) -}}
+  {{- if .pre.svc.manager.ingress.enabled }}
+    {{- $_ := set $autoEnv "KONG_ADMIN_GUI_URL" (include "kong.ingress.serviceUrl" .pre.svc.manager.ingress) -}}
+  {{- end -}}
+
+  {{- if not .pre.enterprise.vitals.enabled }}
+    {{- $_ := set $autoEnv "KONG_VITALS" "off" -}}
+  {{- end }}
+  {{- $_ := set $autoEnv "KONG_CLUSTER_TELEMETRY_LISTEN" (include "kong.listen" .pre.svc.clustertelemetry) -}}
+
+  {{- if .pre.enterprise.portal.enabled }}
+    {{- $_ := set $autoEnv "KONG_PORTAL" "on" -}}
+      {{- $_ := set $autoEnv "KONG_PORTAL_GUI_LISTEN" (include "kong.listen" .pre.svc.portal) -}}
+    {{- $_ := set $autoEnv "KONG_PORTAL_API_LISTEN" (include "kong.listen" .pre.svc.portalapi) -}}
+
+    {{- if .pre.svc.portal.ingress.enabled }}
+      {{- $_ := set $autoEnv "KONG_PORTAL_GUI_HOST" .pre.svc.portal.ingress.hostname -}}
+      {{- if .pre.svc.portal.ingress.tls }}
+        {{- $_ := set $autoEnv "KONG_PORTAL_GUI_PROTOCOL" "https" -}}
+      {{- else }}
+        {{- $_ := set $autoEnv "KONG_PORTAL_GUI_PROTOCOL" "http" -}}
+      {{- end }}
+    {{- end }}
+
+    {{- if .pre.svc.portalapi.ingress.enabled }}
+      {{- $_ := set $autoEnv "KONG_PORTAL_API_URL" (include "kong.ingress.serviceUrl" .pre.svc.portalapi.ingress) -}}
+    {{- end }}
+  {{- end }}
+
+  {{- if .pre.enterprise.rbac.enabled }}
+    {{- $_ := set $autoEnv "KONG_ENFORCE_RBAC" "on" -}}
+    {{- $_ := set $autoEnv "KONG_ADMIN_GUI_AUTH" .pre.enterprise.rbac.admin_gui_auth | default "basic-auth" -}}
+
+    {{- if not (eq .pre.enterprise.rbac.admin_gui_auth "basic-auth") }}
+      {{- $guiAuthConf := include "secretkeyref" (dict "name" .pre.enterprise.rbac.admin_gui_auth_conf_secret "key" "admin_gui_auth_conf") -}}
+      {{- $_ := set $autoEnv "KONG_ADMIN_GUI_AUTH_CONF" $guiAuthConf -}}
+    {{- end }}
+
+    {{- $guiSessionConf := include "secretkeyref" (dict "name" .pre.enterprise.rbac.session_conf_secret "key" "admin_gui_session_conf") -}}
+    {{- $_ := set $autoEnv "KONG_ADMIN_GUI_SESSION_CONF" $guiSessionConf -}}
+  {{- end }}
+
+  {{- if .pre.enterprise.smtp.enabled }}
+    {{- $_ := set $autoEnv "KONG_SMTP_MOCK" "off" -}}
+    {{- $_ := set $autoEnv "KONG_PORTAL_EMAILS_FROM" .pre.enterprise.smtp.portal_emails_from -}}
+    {{- $_ := set $autoEnv "KONG_PORTAL_EMAILS_REPLY_TO" .pre.enterprise.smtp.portal_emails_reply_to -}}
+    {{- $_ := set $autoEnv "KONG_ADMIN_EMAILS_FROM" .pre.enterprise.smtp.admin_emails_from -}}
+    {{- $_ := set $autoEnv "KONG_ADMIN_EMAILS_REPLY_TO" .pre.enterprise.smtp.admin_emails_reply_to -}}
+    {{- $_ := set $autoEnv "KONG_SMTP_ADMIN_EMAILS" .pre.enterprise.smtp.smtp_admin_emails -}}
+    {{- $_ := set $autoEnv "KONG_SMTP_HOST" .pre.enterprise.smtp.smtp_host -}}
+    {{- $_ := set $autoEnv "KONG_SMTP_AUTH_TYPE" .pre.enterprise.smtp.smtp_auth_type -}}
+    {{- $_ := set $autoEnv "KONG_SMTP_SSL" .pre.enterprise.smtp.smtp_ssl -}}
+    {{- $_ := set $autoEnv "KONG_SMTP_PORT" .pre.enterprise.smtp.smtp_port -}}
+    {{- $_ := set $autoEnv "KONG_SMTP_STARTTLS" (quote .pre.enterprise.smtp.smtp_starttls) -}}
+    {{- if .pre.enterprise.smtp.auth.smtp_username }}
+      {{- $_ := set $autoEnv "KONG_SMTP_USERNAME" .pre.enterprise.smtp.auth.smtp_username -}}
+      {{- $smtpPassword := include "secretkeyref" (dict "name" .pre.enterprise.smtp.auth.smtp_password_secret "key" "smtp_password") -}}
+      {{- $_ := set $autoEnv "KONG_SMTP_PASSWORD" $smtpPassword -}}
+    {{- end }}
+  {{- else }}
+    {{- $_ := set $autoEnv "KONG_SMTP_MOCK" "on" -}}
+  {{- end }}
+
+  {{- if .pre.enterprise.license_secret -}}
+    {{- $lic := include "secretkeyref" (dict "name" .pre.enterprise.license_secret "key" "license") -}}
+    {{- $_ := set $autoEnv "KONG_LICENSE_DATA" $lic -}}
+  {{- end }}
+
+{{- end }} {{/* End of the.pre.enterprise settings block */}}
+
+{{- if .pre.postgresql.enabled }}
+  {{- $_ := set $autoEnv "KONG_PG_HOST" (include "kong.pre.postgresql.fullname" .) -}}
+  {{- $_ := set $autoEnv "KONG_PG_PORT" .pre.postgresql.service.ports.pre.postgresql -}}
+  {{- $pgPassword := include "secretkeyref" (dict "name" (include "kong.pre.postgresql.fullname" .) "key" "password") -}}
+
+  {{- $_ := set $autoEnv "KONG_PG_PASSWORD" $pgPassword -}}
+{{- else if eq .env.database "postgres" }}
+  {{- $_ := set $autoEnv "KONG_PG_PORT" "5432" }}
+{{- end }}
+
+{{- if (and (not .pre.ingressControllerEnabled) (eq .env.database "off")) }}
+{{- $dblessSourceCount := (add (.pre.dblessConfig.configMap | len | min 1) (.pre.dblessConfig.secret | len | min 1) (.pre.dblessConfig.config | len | min 1)) -}}
+{{- if eq $dblessSourceCount 1 -}}
+  {{- $_ := set $autoEnv "KONG_DECLARATIVE_CONFIG" "/kong_dbless/kong.yml" -}}
+{{- end }}
+{{- end }}
+
+{{- $_ := set $autoEnv "KONG_PLUGINS" (include "kong.plugins" .) -}}
+
+{{/*
+    ====== USER-SET ENVIRONMENT VARIABLES ======
+*/}}
+
+{{- $userEnv := dict -}}
+{{- range $key, $val := .env }}
+  {{- if (contains "_log" $key) -}}
+    {{- if (eq (typeOf $val) "bool") -}}
+      {{- fail (printf "env.%s must use string 'off' to disable. Without quotes, YAML will coerce the value to a boolean and Kong will reject it" $key) -}}
+	{{- end -}}
+  {{- end -}}
+  {{- $upper := upper $key -}}
+  {{- $var := printf "KONG_%s" $upper -}}
+  {{- $_ := set $userEnv $var $val -}}
+{{- end -}}
+
+{{/*
+    ====== CUSTOM-SET ENVIRONMENT VARIABLES ======
+*/}}
+
+{{- $customEnv := dict -}}
+{{- range $key, $val := .customEnv }}
+  {{- $upper := upper $key -}}
+  {{- $_ := set $customEnv $upper $val -}}
+{{- end -}}
+
+{{/*
+      ====== MERGE AND RENDER ENV BLOCK ======
+*/}}
+
+{{- $completeEnv := mergeOverwrite $autoEnv $userEnv $customEnv -}}
+{{- template "kong.renderEnv" $completeEnv -}}
+
+{{- end -}}
+
+
diff --git a/charts/kong/templates/kong-deploy.yaml b/charts/kong/templates/kong-deploy.yaml
index 76b73ccd8..4c5cb69aa 100644
--- a/charts/kong/templates/kong-deploy.yaml
+++ b/charts/kong/templates/kong-deploy.yaml
@@ -3,6 +3,8 @@
 {{- $deployConfig = (mustMerge $deployConfig .Values.deployment.kong) -}}
 {{- $deployPreRender := dict -}} {{/* collection of values rendered from includes, so that there's no risk of colliding with things from the merge */}}
 {{- $podPreRender := dict -}}
+{{- $containerConfig := .Values.deployment.kong.pod.container -}}
+{{- $containerPreRender := dict -}}
 
 # TODO 921 review how these names end up in practice with a "kong" release. we've historically
 # suggested a lot of example configurations that resulted in various "kong-kong" resources which,
@@ -27,6 +29,8 @@
 # unnecessary volumes doesn't really hurt us
 {{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
 {{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
+{{- $_ := set $containerPreRender "volumeMounts" (include "kong.volumeMounts" .) -}}
+{{- $_ := set $containerPreRender "userVolumeMounts" (include "kong.userDefinedVolumeMounts" .) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "component" "proxy" -}}
@@ -34,12 +38,9 @@
 {{- $_ := set $podPreRender "name" (printf "%s-%s" (include "kong.fullname" .) "proxy") -}}
 {{- $_ := set $podPreRender "versionLabel" .Chart.AppVersion -}}
 
-# TODO 921 rename once the old block is gone
-{{- $containerConfig := .Values.deployment.kong.pod.container -}}
-{{- $containerPre := dict -}}
 {{- $serviceConfig := dict -}}
-{{- $_ := set $containerPre "admissionWebhook" .Values.admissionWebhook -}}
-{{- $_ := set $containerPre "enterprise" .Values.enterprise -}}
+{{- $_ := set $containerPreRender "admissionWebhook" .Values.admissionWebhook -}}
+{{- $_ := set $containerPreRender "enterprise" .Values.enterprise -}}
 
 {{- $_ := set $serviceConfig "admin" .Values.admin -}}
 {{- $_ := set $serviceConfig "proxy" .Values.proxy -}}
@@ -50,12 +51,71 @@
 {{- $_ := set $serviceConfig "portal" .Values.portal -}}
 {{- $_ := set $serviceConfig "portalapi" .Values.portalapi -}}
 {{- $_ := set $serviceConfig "clustertelemetry" .Values.clustertelemetry -}}
-{{- $_ := set $containerPre "svc" $serviceConfig -}}
+{{- $_ := set $containerPreRender "svc" $serviceConfig -}}
+
+{{- $_ := set $containerPreRender "serviceAccount" .Values.deployment.kong.pod.serviceAccount -}}
+{{- $_ := set $containerPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
+{{- $_ := set $containerPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
+
+# TODO 921 there's some duplication here from pre.container, but that allows everything in env
+# that was previously at the root root (.Values) to remain at the new scoped root, rather than
+# having some things in . and some things in .envPre or whatever
+{{- $envRoot := dict -}}
+{{- $_ := set $envRoot "ingressControllerEnabled" .Values.deployment.controller.enabled -}}
+{{- $_ := set $envRoot "svc" $serviceConfig -}}
+{{- $_ := set $envRoot "certificates" .Values.certificates -}}
+{{- $_ := set $envRoot "enterprise" .Values.enterprise -}}
+{{- $_ := set $containerPreRender "envRoot" $envRoot -}}
+# TODO decide which strat to use, separate env or just shove everything into container
+{{- $envRoot := dict -}}
+{{- $_ := set $containerPreRender "ingressControllerEnabled" .Values.deployment.controller.enabled -}}
+{{- $_ := set $containerPreRender "Capabilities" .Capabilities -}}
+{{- $_ := set $containerPreRender "Release" .Release -}}
+{{- $_ := set $containerPreRender "dblessConfig" .Values.dblessConfig -}}
+{{- $_ := set $containerPreRender "plugins" .Values.plugins -}}
+# TODO 921 we should probably take this opportunity to drop the postgresql subchart. keeping for
+# now to try and reduce scope (hah)
+{{- $_ := set $containerPreRender "postgresql" .Values.postgresql -}}
+
+# TODO 921 figure this initcontainer out. it needs stuff like env and volumemounts figured out
+# first, and then needs to be injected into a prerender var, since we don't know whether
+# we're rendering kong or the controller here
+{{/*
+      {{- if .Values.deployment.kong.enabled }}
+      initContainers:
+      - name: clear-stale-pid
+        image: {{ include "kong.getRepoTag" .Values.image }}
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        securityContext:
+        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        resources:
+{{ toYaml .Values.resources | indent 10 }}
+        command:
+        - "rm"
+        - "-vrf"
+        - "$KONG_PREFIX/pids"
+        env:
+        {{- include "kong.env" . | nindent 8 }}
+        volumeMounts:
+        {{- include "kong.volumeMounts" . | nindent 8 }}
+        {{- if .Values.deployment.initContainers }}
+        {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+        {{- end }}
+        {{- if (and (not (eq .Values.env.database "off")) .Values.waitImage.enabled) }}
+        {{- include "kong.wait-for-db" . | nindent 6 }}
+        {{- end }}
+      {{- end }} # TODO 921 END initContainers clear-stale-pid
+*/}}
+{{- $initContainers := list -}}
+{{- range .Values.deployment.kong.pod.initContainers -}}
+  {{- $initContainers = mustAppend $initContainers . -}}
+{{- end -}}
+{{- $_ := set $podPreRender "initContainers" $initContainers -}}
+
+{{- $volumeMounts := list -}}
+{{- $volumeMounts = mustAppend $volumeMounts (dict "foo" "bar") -}}
 
-{{- $_ := set $containerPre "serviceAccount" .Values.deployment.kong.pod.serviceAccount -}}
-{{- $_ := set $containerPre "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
-{{- $_ := set $containerPre "serviceAccountName" (include "kong.serviceAccountName" .) -}}
-{{- $_ := set $containerConfig "pre" $containerPre -}}
+{{- $_ := set $containerConfig "pre" $containerPreRender -}}
 
 {{- $_ := set $podPreRender "container" (include "kong.proxy-container-new" $containerConfig ) -}}
 
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 48c3f8b18..93226fbc9 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -151,6 +151,7 @@ deployment:
           periodSeconds: 10
           successThreshold: 1
           failureThreshold: 3
+        lifecycle: {}
         securityContext:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
@@ -193,9 +194,6 @@ deployment:
       # rollingUpdate:
       #   maxSurge: "100%"
       #   maxUnavailable: "0%"
-    sidecarContainers: []
-    userDefinedVolumes: []
-    initContainers: []
     pod:
       annotations: {} # .podAnnotations, for the controller
       # TODO 921 this should probably go elsewhere
@@ -225,6 +223,9 @@ deployment:
       # Ref: https://kubernetes.io/docs/user-guide/node-selection/
       nodeSelector: {}
       securityContext: {}
+      sidecarContainers: []
+      userDefinedVolumes: []
+      initContainers: []
       container:
         args: []
         env:
@@ -272,6 +273,15 @@ deployment:
           periodSeconds: 10
           successThreshold: 1
           failureThreshold: 3
+        lifecycle:
+          preStop:
+            exec:
+              # kong quit has a default timeout of 10 seconds, and a default wait of 0 seconds.
+              # Note: together they should be less than the terminationGracePeriodSeconds setting below.
+              command:
+                - kong
+                - quit
+                - '--wait=15'
         securityContext:
           readOnlyRootFilesystem: true
           allowPrivilegeEscalation: false
@@ -282,7 +292,9 @@ deployment:
           capabilities:
             drop:
             - ALL
-        userDefinedVolumeMounts: []
+        userDefinedVolumeMounts:
+        - name: webhook-cert
+          mountPath: /example
         # If you want to specify resources, uncomment the following
         # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
         resources: {}
@@ -882,8 +894,8 @@ ingressController:
   # kubernetes resources, so it doesn't necessarily fit under the `deployment`
   # section.
   gatewayDiscovery:
-    enabled: false
-    generateAdminApiService: false
+    enabled: true
+    generateAdminApiService: true
     adminApiService:
       namespace: ""
       name: ""

From 8f14b936ac11f40df2212cb1dcb66cc1496fd880 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Fri, 10 Nov 2023 15:35:58 -0800
Subject: [PATCH 21/24] wip: fix template, fix discovery

---
 charts/kong/templates/env.yaml | 101 +++++++++++++++++++++++++++++++++
 charts/kong/values.yaml        |   4 +-
 2 files changed, 103 insertions(+), 2 deletions(-)

diff --git a/charts/kong/templates/env.yaml b/charts/kong/templates/env.yaml
index b5ce7f0fb..9bda26d0d 100644
--- a/charts/kong/templates/env.yaml
+++ b/charts/kong/templates/env.yaml
@@ -278,3 +278,104 @@ the template that it itself is using form the above sections.
 {{- end -}}
 
 
+{{/* =========== CONTROLLER ENVIRONMENT =========== */}}
+
+# TODO 921 everything here would need to become relative. some of it relies on
+# external values though, e.g. .Values.ingressController.ingressClass and
+# .Values.ingressController.admissionWebhook.address presumably remain where
+# they are.
+{{- define "kong.env.controller" -}}
+{{/*
+    ====== AUTO-GENERATED ENVIRONMENT VARIABLES ======
+*/}}
+
+
+{{- $autoEnv := dict -}}
+  {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY" true -}}
+  {{- $_ := set $autoEnv "CONTROLLER_PUBLISH_SERVICE" ( include "kong.controller-publish-service" . ) -}}
+  {{- $_ := set $autoEnv "CONTROLLER_INGRESS_CLASS" .Values.ingressController.ingressClass -}}
+  {{- $_ := set $autoEnv "CONTROLLER_ELECTION_ID" (printf "kong-ingress-controller-leader-%s" .Values.ingressController.ingressClass) -}}
+
+  {{- if .Values.ingressController.admissionWebhook.enabled }}
+    {{- $address := (default "0.0.0.0" .Values.ingressController.admissionWebhook.address) -}}
+    {{- $_ := set $autoEnv "CONTROLLER_ADMISSION_WEBHOOK_LISTEN" (printf "%s:%d" $address (int64 .Values.ingressController.admissionWebhook.port)) -}}
+  {{- end }}
+  {{- if (not (eq (len .Values.ingressController.watchNamespaces) 0)) }}
+    {{- $_ := set $autoEnv "CONTROLLER_WATCH_NAMESPACE" (.Values.ingressController.watchNamespaces | join ",") -}}
+  {{- end }}
+
+{{/*
+    ====== ADMIN API CONFIGURATION ======
+*/}}
+
+  {{- if .Values.ingressController.gatewayDiscovery.enabled -}}
+    # TODO 921 this will need to be an pre var, it's not available from controller scope
+    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_SVC" (printf "%s/%s-%s" (include "kong.namespace" .) (include "kong.fullname" .) "admin") -}}
+  {{- else -}}
+    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_URL" (include "kong.adminLocalURL" .) -}}
+  {{- end -}}
+
+  {{- if .Values.ingressController.adminApi.tls.client.enabled }}
+    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_CLIENT_CERT_FILE" "/etc/secrets/admin-api-cert/tls.crt" -}}
+    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_CLIENT_KEY_FILE" "/etc/secrets/admin-api-cert/tls.key" -}}
+  {{- end }}
+
+{{/*
+    ====== KONNECT ENVIRONMENT VARIABLES ======
+*/}}
+
+{{- if .Values.ingressController.konnect.enabled }}
+  {{- if (semverCompare "< 2.9.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+  {{- fail (printf "Konnect sync is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+  {{- end }}
+
+  {{- if not .Values.ingressController.gatewayDiscovery.enabled }}
+  {{- fail "ingressController.gatewayDiscovery.enabled has to be true when ingressController.konnect.enabled"}}
+  {{- end }}
+
+  {{- $konnect := .Values.ingressController.konnect -}}
+  {{- $_ := required "ingressController.konnect.runtimeGroupID is required when ingressController.konnect.enabled" $konnect.runtimeGroupID -}}
+
+  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_SYNC_ENABLED" true -}}
+  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_RUNTIME_GROUP_ID" $konnect.runtimeGroupID -}}
+  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_ADDRESS" (printf "https://%s" .Values.ingressController.konnect.apiHostname) -}}
+
+  {{- $tlsCert := include "secretkeyref" (dict "name" $konnect.tlsClientCertSecretName "key" "tls.crt") -}}
+  {{- $tlsKey := include "secretkeyref" (dict "name" $konnect.tlsClientCertSecretName "key" "tls.key") -}}
+  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_TLS_CLIENT_CERT" $tlsCert -}}
+  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_TLS_CLIENT_KEY" $tlsKey -}}
+
+  {{- if $konnect.license.enabled }}
+  {{- $_ = set $autoEnv "CONTROLLER_KONNECT_LICENSING_ENABLED" true -}}
+  {{- end }}
+{{- end }}
+
+{{/*
+    ====== USER-SET ENVIRONMENT VARIABLES ======
+*/}}
+
+{{- $userEnv := dict -}}
+{{- range $key, $val := .Values.deployment.controller.pod.container.env }}
+  {{- $upper := upper $key -}}
+  {{- $var := printf "CONTROLLER_%s" $upper -}}
+  {{- $_ := set $userEnv $var $val -}}
+{{- end -}}
+
+{{/*
+    ====== CUSTOM-SET INGRESS CONTROLLER ENVIRONMENT VARIABLES ======
+*/}}
+
+{{- $customIngressEnv := dict -}}
+{{- range $key, $val := .Values.ingressController.customEnv }}
+  {{- $upper := upper $key -}}
+  {{- $_ := set $customIngressEnv $upper $val -}}
+{{- end -}}
+
+{{/*
+      ====== MERGE AND RENDER ENV BLOCK ======
+*/}}
+
+{{- $completeEnv := mergeOverwrite $autoEnv $userEnv $customIngressEnv -}}
+{{- template "kong.renderEnv" $completeEnv -}}
+
+{{- end -}}
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 93226fbc9..cdb487c62 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -473,8 +473,8 @@ admin:
   # Enable creating a Kubernetes service for the admin API
   # Disabling this is recommended for most ingress controller configurations
   # Enterprise users that wish to use Kong Manager with the controller should enable this
-  enabled: false
-  type: NodePort
+  enabled: true
+  type: ClusterIP
   loadBalancerClass:
   # To specify annotations or labels for the admin service, add them to the respective
   # "annotations" or "labels" dictionaries below.

From 6ad04bb494374c5cf76e624b73ce37a93280cd2e Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Fri, 10 Nov 2023 16:49:58 -0800
Subject: [PATCH 22/24] wip: many old things removed, migrations broken

---
 charts/kong/templates/_helpers.tpl            |  14 +-
 .../kong/templates/deployment-controller.yaml |   6 +-
 charts/kong/templates/env.yaml                |  17 +-
 charts/kong/templates/kong-deploy.yaml        |   2 +
 charts/kong/templates/secret-sa-token.yaml    |   2 +-
 charts/kong/templates/service-account.yaml    |   6 +-
 charts/kong/templates/tests/test-jobs.yaml    |   2 +-
 .../kong/templates/tests/test-resources.yaml  |   2 +-
 charts/kong/values.yaml                       | 473 ++++--------------
 9 files changed, 114 insertions(+), 410 deletions(-)

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index 822fb3e70..b721799d5 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -62,10 +62,10 @@ app.kubernetes.io/instance: "{{ .Release.Name }}"
 Create the name of the service account to use
 */}}
 {{- define "kong.serviceAccountName" -}}
-{{- if .Values.deployment.serviceAccount.create -}}
-    {{ default (include "kong.fullname" .) .Values.deployment.serviceAccount.name }}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "kong.fullname" .) .Values.serviceAccount.name }}
 {{- else -}}
-    {{ default "default" .Values.deployment.serviceAccount.name }}
+    {{ default "default" .Values.serviceAccount.name }}
 {{- end -}}
 {{- end -}}
 
@@ -476,11 +476,11 @@ The name of the Service which will be used by the controller to update the Ingre
 {{- define "kong.volumes" -}}
 - name: {{ template "kong.fullname" . }}-prefix-dir
   emptyDir:
-    sizeLimit: {{ .Values.deployment.prefixDir.sizeLimit }}
+    sizeLimit: {{ .Values.deployment.kong.pod.container.prefixDir.sizeLimit }}
 - name: {{ template "kong.fullname" . }}-tmp
   emptyDir:
-    sizeLimit: {{ .Values.deployment.tmpDir.sizeLimit }}
-{{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+    sizeLimit: {{ .Values.deployment.kong.pod.container.tmpDir.sizeLimit }}
+{{- if (and (not .Values.serviceAccount.automountServiceAccountToken) (or .Values.serviceAccount.create .Values.serviceAccount.name)) }}
 - name: {{ template "kong.serviceAccountTokenName" . }}
   {{- /* Due to GKE versions (e.g. v1.23.15-gke.1900) we need to handle pre-release part of the version as well.
   See the related documentation of semver module that Helm depends on for semverCompare:
@@ -824,7 +824,7 @@ The name of the Service which will be used by the controller to update the Ingre
     mountPath: /admission-webhook
     readOnly: true
 {{- end }}
-{{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+{{- if (and (not .Values.serviceAccount.automountServiceAccountToken) (or .Values.serviceAccount.create .Values.serviceAccount.name)) }}
   - name: {{ template "kong.serviceAccountTokenName" . }}
     mountPath: /var/run/secrets/kubernetes.io/serviceaccount
     readOnly: true
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index 24b8ee820..cd2de4ae8 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -42,10 +42,10 @@ spec:
   template:
     metadata:
       annotations:
-        {{/* # TODO 921 some keys in the pod template end up being from the deployment dict, so probably no separate define 
+        {{/* # NOTE 921 some keys in the pod template end up being from the deployment dict, so probably no separate define 
              # there's no obvious reason to separate them, since there can only be one per Deployment anyway
         */ -}}
-        {{- if (and (not .pod.serviceAccount.automountServiceAccountToken) (or .pod.serviceAccount.create .pod.serviceAccount.name)) }}
+        {{- if (and (not .pod.automountServiceAccountToken) (or .pre.deployment.serviceAccount.create .pre.deployment.serviceAccount.name)) }}
         {{/* # TODO 921 needs to be passed in as-is. possibly should be made relative, since the generator will attempt to use the same
              # name, but lives in a separate service-account.yaml template, so vov
              # TODO above is complete but keeping the note around for now
@@ -88,7 +88,7 @@ spec:
       automountServiceAccountToken: false
       {{ end }}
       {{- if .pod.container.image.pullSecrets }}
-      # TODO 921 this is out of place, but the rest of .image _does_ live
+      # NOTE 921 this is out of place, but the rest of .image _does_ live
       # inside the container, so probably just keep it there, rather than
       # separating into .pod.imagePullSecrets
       imagePullSecrets:
diff --git a/charts/kong/templates/env.yaml b/charts/kong/templates/env.yaml
index 9bda26d0d..a2f3b82a1 100644
--- a/charts/kong/templates/env.yaml
+++ b/charts/kong/templates/env.yaml
@@ -308,12 +308,12 @@ the template that it itself is using form the above sections.
     ====== ADMIN API CONFIGURATION ======
 */}}
 
-  {{- if .Values.ingressController.gatewayDiscovery.enabled -}}
-    # TODO 921 this will need to be an pre var, it's not available from controller scope
-    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_SVC" (printf "%s/%s-%s" (include "kong.namespace" .) (include "kong.fullname" .) "admin") -}}
-  {{- else -}}
-    {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_URL" (include "kong.adminLocalURL" .) -}}
-  {{- end -}}
+  # TODO 921 this will need to be an pre var, it's not available from controller scope
+  # Support for automatic CONTROLLER_KONG_ADMIN_URL has been removed, but we somehow need to handle DB-backed instances that need to
+  # set it manually: some installs are weird and split the Kong instance into an entire separate cluster, or something that's
+  # otherwise not available via a local Service.
+  # The controller code gives ADMIN_SVC precedence, so if we set it without checking it will just knock out the user config.
+  {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_SVC" (printf "%s/%s-%s" (include "kong.namespace" .) (include "kong.fullname" .) "admin") -}}
 
   {{- if .Values.ingressController.adminApi.tls.client.enabled }}
     {{- $_ := set $autoEnv "CONTROLLER_KONG_ADMIN_TLS_CLIENT_CERT_FILE" "/etc/secrets/admin-api-cert/tls.crt" -}}
@@ -329,10 +329,7 @@ the template that it itself is using form the above sections.
   {{- fail (printf "Konnect sync is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
   {{- end }}
 
-  {{- if not .Values.ingressController.gatewayDiscovery.enabled }}
-  {{- fail "ingressController.gatewayDiscovery.enabled has to be true when ingressController.konnect.enabled"}}
-  {{- end }}
-
+  # TODO 921 removed a "must set GW discovery" check here since the setting will go away. dunno if we bother with a check on admin URL setting
   {{- $konnect := .Values.ingressController.konnect -}}
   {{- $_ := required "ingressController.konnect.runtimeGroupID is required when ingressController.konnect.enabled" $konnect.runtimeGroupID -}}
 
diff --git a/charts/kong/templates/kong-deploy.yaml b/charts/kong/templates/kong-deploy.yaml
index 4c5cb69aa..cf75f71ab 100644
--- a/charts/kong/templates/kong-deploy.yaml
+++ b/charts/kong/templates/kong-deploy.yaml
@@ -22,8 +22,10 @@
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
+# TODO 921 SA stuff can probably be refactored to be cleaner, but as-is it's functional
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
+{{- $_ := set $deployPreRender "serviceAccount" .Values.serviceAccount -}}
 # TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
 # pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
 # unnecessary volumes doesn't really hurt us
diff --git a/charts/kong/templates/secret-sa-token.yaml b/charts/kong/templates/secret-sa-token.yaml
index fe8a67d23..e1ff34952 100644
--- a/charts/kong/templates/secret-sa-token.yaml
+++ b/charts/kong/templates/secret-sa-token.yaml
@@ -2,7 +2,7 @@
 See the related documentation of semver module that Helm depends on for semverCompare:
 https://github.com/Masterminds/semver#working-with-prerelease-versions
 Related Helm issue: https://github.com/helm/helm/issues/3810 */}}
-{{- if and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) (semverCompare "<1.20.0-0" .Capabilities.KubeVersion.Version) }}
+{{- if and (or .Values.serviceAccount.create .Values.serviceAccount.name) (semverCompare "<1.20.0-0" .Capabilities.KubeVersion.Version) }}
 apiVersion: v1
 kind: Secret
 metadata:
diff --git a/charts/kong/templates/service-account.yaml b/charts/kong/templates/service-account.yaml
index 68afcf031..268c6d27d 100644
--- a/charts/kong/templates/service-account.yaml
+++ b/charts/kong/templates/service-account.yaml
@@ -1,12 +1,12 @@
-{{- if and (or .Values.deployment.kong.enabled .Values.deployment.controller.enabled) .Values.deployment.controller.pod.serviceAccount.create -}}
+{{- if and (or .Values.deployment.kong.enabled .Values.deployment.controller.enabled) .Values.serviceAccount.create -}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{ template "kong.serviceAccountName" . }}
   namespace: {{ template "kong.namespace" . }}
-  {{- if .Values.deployment.serviceAccount.annotations }}
+  {{- if .Values.serviceAccount.annotations }}
   annotations:
-  {{- range $key, $value := .Values.deployment.serviceAccount.annotations }}
+  {{- range $key, $value := .Values.serviceAccount.annotations }}
     {{ $key }}: {{ $value | quote }}
   {{- end }}
   {{- end }}
diff --git a/charts/kong/templates/tests/test-jobs.yaml b/charts/kong/templates/tests/test-jobs.yaml
index d34660ee8..5d0dc9be8 100644
--- a/charts/kong/templates/tests/test-jobs.yaml
+++ b/charts/kong/templates/tests/test-jobs.yaml
@@ -1,4 +1,4 @@
-{{- if  .Values.deployment.test.enabled }}
+{{- if  .Values.test.enabled }}
 ---
 apiVersion: v1
 kind: Pod
diff --git a/charts/kong/templates/tests/test-resources.yaml b/charts/kong/templates/tests/test-resources.yaml
index 79e32d7e2..9630ce312 100644
--- a/charts/kong/templates/tests/test-resources.yaml
+++ b/charts/kong/templates/tests/test-resources.yaml
@@ -1,4 +1,4 @@
-{{- if  .Values.deployment.test.enabled }}
+{{- if  .Values.test.enabled }}
 ---
 apiVersion: v1
 kind: Pod
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index cdb487c62..20305b5f5 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -39,7 +39,6 @@
 #     autoscaling: {} # A duplicate of the current root key, for the controller Deployment. This may actually be a separate autoscaling.controller section.
 #     pod:
 #       annotations: {} # Pod-level annotations. currently there is a single root-level .podAnnotations used for the single unified Pod
-#       serviceAccount: {}
 #       container:
 #         env: {...} # controller environment
 #         image: {}
@@ -67,30 +66,19 @@ deployment:
     # base not set defaults in these cases, and then having a dedicated example
     # values.yaml that just shows structure.
     autoscaling: {}
-    # TODO 921 parity, though it really makes _no_ sense for a daemonset
-    # controller. that said, we may be able to use the same templates for both
-    # Deployments' basic Deployment-y things and get it for free.
-    daemonset: false
     updateStrategy: {}
       # type: RollingUpdate
       # rollingUpdate:
       #   maxSurge: "100%"
       #   maxUnavailable: "0%"
-    sidecarContainers: []
-    userDefinedVolumes: []
-    initContainers: []
+    replicaCount: 1
+    ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing,
+    ## for it to be considered available.
+    # minReadySeconds: 60
     pod:
+      labels: {}
       annotations: {} # .podAnnotations, for the controller
-      # TODO 921 this should probably go elsewhere
-      serviceAccount:
-        create: true
-        # TODO 921 we arguably don't need this with a split controller
-        # Deployment, since we use it to hide the token from Kong and its
-        # arbitrary plugin codeTODO 921 we arguably don't need this with a
-        # split controller Deployment, since we use it to hide the token from
-        # Kong and its arbitrary plugin code.
-        # TODO 921 probably ignore the above, I think we still need it for the Kuma case
-        automountServiceAccountToken: false
+      automountServiceAccountToken: false
       hostNetwork: false
       # Affinity for pod assignment
       # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
@@ -107,7 +95,25 @@ deployment:
       # Node labels for pod assignment
       # Ref: https://kubernetes.io/docs/user-guide/node-selection/
       nodeSelector: {}
+      priorityClassName: ""
       securityContext: {}
+      userDefinedVolumes: []
+      hostAliases: []
+      initContainers: []
+      sidecarContainers: []
+      ## Optional DNS configuration for pods
+      # dnsPolicy: ClusterFirst
+      # dnsConfig:
+      #   nameservers:
+      #   - "10.100.0.10"
+      #   options:
+      #   - name: ndots
+      #     value: "5"
+      #   searches:
+      #   - default.svc.cluster.local
+      #   - svc.cluster.local
+      #   - cluster.local
+      #   - us-east-1.compute.internal
       container:
         args: []
         env:
@@ -122,6 +128,9 @@ deployment:
           #     secretKeyRef:
           #        name: CHANGEME-admin-token-secret
           #        key: CHANGEME-admin-token-key
+        # Additional non-Kong environment variables. Keys here will be set in
+        # the container environment _without_ prepending a "CONTROLLER_" prefix.
+        # customEnv: {}
         image:
           repository: kong/kubernetes-ingress-controller
           tag: "2.12"
@@ -172,6 +181,11 @@ deployment:
           # requests:
           #  cpu: 1
           #  memory: 2G
+        # TODO 921 since kong.volumes isn't properly split yet, this
+        # is not honored (it uses the setting under deployment.kong)
+        # tmp empty dir size
+        tmpDir:
+          sizeLimit: 1Gi
 
   kong:
     enabled: true
@@ -185,27 +199,22 @@ deployment:
     # base not set defaults in these cases, and then having a dedicated example
     # values.yaml that just shows structure.
     autoscaling: {}
-    # TODO 921 parity, though it really makes _no_ sense for a daemonset
-    # controller. that said, we may be able to use the same templates for both
-    # Deployments' basic Deployment-y things and get it for free.
     daemonset: false
     updateStrategy: {}
       # type: RollingUpdate
       # rollingUpdate:
       #   maxSurge: "100%"
       #   maxUnavailable: "0%"
+    replicaCount: 1
+    ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing,
+    ## for it to be considered available.
+    # minReadySeconds: 60
     pod:
-      annotations: {} # .podAnnotations, for the controller
-      # TODO 921 this should probably go elsewhere
-      serviceAccount:
-        create: true
-        # TODO 921 we arguably don't need this with a split controller
-        # Deployment, since we use it to hide the token from Kong and its
-        # arbitrary plugin codeTODO 921 we arguably don't need this with a
-        # split controller Deployment, since we use it to hide the token from
-        # Kong and its arbitrary plugin code.
-        # TODO 921 probably ignore the above, I think we still need it for the Kuma case
-        automountServiceAccountToken: false
+      labels: {}
+      annotations:
+        kuma.io/gateway: enabled
+        traffic.sidecar.istio.io/includeInboundPorts: ""
+      automountServiceAccountToken: false
       hostNetwork: false
       # Affinity for pod assignment
       # Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
@@ -222,12 +231,34 @@ deployment:
       # Node labels for pod assignment
       # Ref: https://kubernetes.io/docs/user-guide/node-selection/
       nodeSelector: {}
+      hostAliases: []
+      priorityClassName: ""
       securityContext: {}
       sidecarContainers: []
-      userDefinedVolumes: []
       initContainers: []
+      ## Optional DNS configuration for pods
+      # dnsPolicy: ClusterFirst
+      # dnsConfig:
+      #   nameservers:
+      #   - "10.100.0.10"
+      #   options:
+      #   - name: ndots
+      #     value: "5"
+      #   searches:
+      #   - default.svc.cluster.local
+      #   - svc.cluster.local
+      #   - cluster.local
+      #   - us-east-1.compute.internal
       container:
         args: []
+        # Specify Kong configuration
+        # This chart takes all entries defined under `.env` and transforms them into into `KONG_*`
+        # environment variables for Kong containers.
+        # Their names here should match the names used in https://github.com/Kong/kong/blob/master/kong.conf.default
+        # See https://docs.konghq.com/latest/configuration also for additional details
+        # Values here take precedence over values from other sections of values.yaml,
+        # e.g. setting pg_user here will override the value normally set when postgresql.enabled
+        # is set below. In general, you should not set values here if they are set elsewhere.
         env:
           database: "off"
           router_flavor: "traditional"
@@ -241,6 +272,9 @@ deployment:
           admin_gui_error_log: /dev/stderr
           portal_api_error_log: /dev/stderr
           prefix: /kong_prefix/
+        # Additional non-Kong environment variables. Keys here will be set in
+        # the container environment _without_ prepending a "KONG_" prefix.
+        # customEnv: {}
         image:
           repository: kong
           tag: "3.4"
@@ -304,70 +338,23 @@ deployment:
           # requests:
           #  cpu: 1
           #  memory: 2G
+        # kong_prefix empty dir size
+        prefixDir:
+          sizeLimit: 256Mi
+        # tmp empty dir size
+        tmpDir:
+          sizeLimit: 1Gi
 
-  oldkong:
-    # Enable or disable Kong itself
-    # Setting this to false with ingressController.enabled=true will create a
-    # controller-only release.
-    enabled: true
-  ## Minimum number of seconds for which a newly created pod should be ready without any of its container crashing,
-  ## for it to be considered available.
-  # minReadySeconds: 60
-  # TODO 921 Kong presumably no longer needs a service account at all.
-  ## Specify the service account to create and to be assigned to the deployment / daemonset and for the migrations
-  serviceAccount:
-    create: true
-    # Automount the service account token. By default, this is disabled, and the token is only mounted on the controller
-    # container. Some sidecars require enabling this. Note that enabling this exposes Kubernetes credentials to Kong
-    # Lua code, increasing potential attack surface.
-    automountServiceAccountToken: false
-  ## Optionally specify the name of the service account to create and the annotations to add.
-  #  name:
-  #  annotations: {}
-
-  # TODO 921 pod section
-  ## Optionally specify any extra sidecar containers to be included in the deployment
-  ## See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core
-  # sidecarContainers:
-  #   - name: sidecar
-  #     image: sidecar:latest
-  # initContainers:
-  # - name: initcon
-  #   image: initcon:latest
-  # hostAliases:
-  # - ip: "127.0.0.1"
-  #   hostnames:
-  #   - "foo.local"
-  #   - "bar.local"
-
-  # TODO 921 pod section
-  ## Define any volumes and mounts you want present in the Kong proxy container
-  # userDefinedVolumes:
-  # - name: "volumeName"
-  #   emptyDir: {}
-  # TODO 921 these would live inside the container section
-  # userDefinedVolumeMounts:
-  # - name: "volumeName"
-  #   mountPath: "/opt/user/dir/mount"
-  test:
-    # Enable creation of test resources for use with "helm test"
-    enabled: false
-  # Use a DaemonSet controller instead of a Deployment controller
-  daemonset: false
-  hostNetwork: false
-  # TODO 921 part of the container section. this is kong-specific functionality
-  # though--dunno if we want to handle stuff that isn't generic to either 
-  # Deployment's container differently than stuff that does
-  # kong_prefix empty dir size
-  prefixDir:
-    sizeLimit: 256Mi
-  # tmp empty dir size
-  tmpDir:
-    sizeLimit: 1Gi
 # Override namepsace for Kong chart resources. By default, the chart creates resources in the release namespace.
 # This may not be desirable when using this chart as a dependency.
 # namespace: "example"
 
+serviceAccount:
+  create: true
+  ## Optionally specify the name of the service account to create and the annotations to add.
+  name: ""
+  annotations: {}
+
 # -----------------------------------------------------------------------------
 # Admission webhook
 # -----------------------------------------------------------------------------
@@ -391,74 +378,15 @@ admissionWebhook:
   # Tune the default Kubernetes timeoutSeconds of 10 seconds
   # timeoutSeconds: 10
 
-# -----------------------------------------------------------------------------
-# Kong parameters
-# -----------------------------------------------------------------------------
-
-# Specify Kong configuration
-# This chart takes all entries defined under `.env` and transforms them into into `KONG_*`
-# environment variables for Kong containers.
-# Their names here should match the names used in https://github.com/Kong/kong/blob/master/kong.conf.default
-# See https://docs.konghq.com/latest/configuration also for additional details
-# Values here take precedence over values from other sections of values.yaml,
-# e.g. setting pg_user here will override the value normally set when postgresql.enabled
-# is set below. In general, you should not set values here if they are set elsewhere.
-# TODO 921 deployments.kong.pod.container.env
-env:
-  database: "off"
-  # the chart uses the traditional router (for Kong 3.x+) because the ingress
-  # controller generates traditional routes. if you do not use the controller,
-  # you may set this to "traditional_compatible" or "expressions" to use the new
-  # DSL-based router
-  router_flavor: "traditional"
-  nginx_worker_processes: "2"
-  proxy_access_log: /dev/stdout
-  admin_access_log: /dev/stdout
-  admin_gui_access_log: /dev/stdout
-  portal_api_access_log: /dev/stdout
-  proxy_error_log: /dev/stderr
-  admin_error_log: /dev/stderr
-  admin_gui_error_log: /dev/stderr
-  portal_api_error_log: /dev/stderr
-  prefix: /kong_prefix/
-
-# This section is any customer specific environments variables that doesn't require KONG_ prefix.
-# These custom environment variables are typicall used in custom plugins or serverless plugins to
-# access environment specific credentials or tokens.
-# Example as below, uncomment if required and add additional attributes as required.
-# Note that these environment variables will only apply to the proxy and init container. The ingress-controller
-# container has its own customEnv section.
-
-# TODO 921 deployments.kong.pod.container.customEnv
-# customEnv:
-#   api_token:
-#     valueFrom:
-#       secretKeyRef:
-#         key: token
-#         name: api_key
-#   client_name: testClient
-
 # This section can be used to configure some extra labels that will be added to each Kubernetes object generated.
 extraLabels: {}
 
-# TODO 921 deployments.kong.pod.container.image
-# Specify Kong's Docker image and repository details here
-image:
-  repository: kong
-  tag: "3.4"
-  # Kong Enterprise
-  # repository: kong/kong-gateway
-  # tag: "3.4"
-
-  # Specify a semver version if your image tag is not one (e.g. "nightly")
-  effectiveSemver:
-  pullPolicy: IfNotPresent
-  ## Optionally specify an array of imagePullSecrets.
-  ## Secrets must be manually created in the namespace.
-  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
-  ##
-  # pullSecrets:
-  #   - myRegistrKeySecretName
+########################
+#
+# Service configuration
+#
+########################
+
 
 # TODO 921 we can probably leave the service definitions at the root. Although
 # they're specific to a particular Deployment, they can't be defined
@@ -747,10 +675,6 @@ udpProxy:
     #   # "ssl" is required for SNI-based routes. It is not supported on versions <2.0
     #   parameters: []
 
-# TODO 921 these go into the Kong deployment but probably don't need to move
-# there's no controller equivalent we'd ever need controller/kong separate
-# config.
-#
 # Custom Kong plugins can be loaded into Kong by mounting the plugin code
 # into the file-system of Kong container.
 # The plugin code should be present in ConfigMap or Secret inside the same
@@ -772,21 +696,6 @@ plugins: {}
   # - pluginName: rewriter
   #   name: kong-plugin-rewriter
 
-# TODO 921 We could maybe convert these into standard user mounts? a basic
-# rearrange migration doesn't work, but you can basically handle these by
-# creating a defined volume that fills in the automatic fields:
-#
-# userDefinedVolumes:
-# - name: "<secret-name>"
-#   secret:
-#     name: whatever
-# userDefinedVolumeMounts:
-# - name: "<secret-name>"
-#   mountPath: "/etc/secrets/secret-name"
-#
-# if we leave it as-is, probably no reason to port it to the controller
-# deployment--we can just require creating the full mount config there.
-
 # Inject specified secrets as a volume in Kong Container at path /etc/secrets/{secret-name}/
 # This can be used to override default SSL certificates.
 # Be aware that the secret name will be used verbatim, and that certain types
@@ -873,34 +782,7 @@ dblessConfig:
 # Kong Ingress Controller's primary purpose is to satisfy Ingress resources
 # created in k8s.  It uses CRDs for more fine grained control over routing and
 # for Kong specific configuration.
-# TODO 921 deployments.controller.enabled
 ingressController:
-  enabled: true
-  # TODO 921 deployments.controller.pod.container.image
-  image:
-    repository: kong/kubernetes-ingress-controller
-    tag: "2.12"
-    # Optionally set a semantic version for version-gated features. This can normally
-    # be left unset. You only need to set this if your tag is not a semver string,
-    # such as when you are using a "next" tag. Set this to the effective semantic
-    # version of your tag: for example if using a "next" image for an unreleased 3.1.0
-    # version, set this to "3.1.0".
-    effectiveSemver:
-  # TODO 921 deployments.controller.pod.container.args
-  args: []
-
-  # TODO 921 probably goes under a root-level controller field, which is, well,
-  # where it is now. this is config for the controller _not_ related to its
-  # kubernetes resources, so it doesn't necessarily fit under the `deployment`
-  # section.
-  gatewayDiscovery:
-    enabled: true
-    generateAdminApiService: true
-    adminApiService:
-      namespace: ""
-      name: ""
-
-  # TODO 921 app-level config, probably remains here
   # Specify individual namespaces to watch for ingress configuration. By default,
   # when no namespaces are set, the controller watches all namespaces and uses a
   # ClusterRole to grant access to Kubernetes resources. When you list specific
@@ -910,28 +792,6 @@ ingressController:
   # Requires controller 2.0.0 or newer.
   watchNamespaces: []
 
-  # Specify Kong Ingress Controller configuration via environment variables
-  # TODO 921 deployments.controller.pod.container.env
-  env:
-    # The controller disables TLS verification by default because Kong
-    # generates self-signed certificates by default. Set this to false once you
-    # have installed CA-signed certificates.
-    kong_admin_tls_skip_verify: true
-    # If using Kong Enterprise with RBAC enabled, uncomment the section below
-    # and specify the secret/key containing your admin token.
-    # kong_admin_token:
-    #   valueFrom:
-    #     secretKeyRef:
-    #        name: CHANGEME-admin-token-secret
-    #        key: CHANGEME-admin-token-key
-
-  # This section is any customer specific environments variables that doesn't require CONTROLLER_ prefix.
-  # Example as below, uncomment if required and add additional attributes as required.
-  # customEnv:
-  #   TZ: "Europe/Berlin"
-
-  # TODO 921 separate resource config, probably remains here. only loosely-tied
-  # to the deployment through a service we generate for you.
   admissionWebhook:
     enabled: true
     failurePolicy: Ignore
@@ -951,57 +811,14 @@ ingressController:
     # Tune the default Kubernetes timeoutSeconds of 10 seconds
     # timeoutSeconds: 10
 
-  # TODO 921 app-level config, probably remains here
   ingressClass: kong
   # annotations for IngressClass resource (Kubernetes 1.18+)
   ingressClassAnnotations: {}
 
-  ## Define any volumes and mounts you want present in the ingress controller container
-  ## Volumes are defined above in deployment.userDefinedVolumes
-  # userDefinedVolumeMounts:
-  # - name: "volumeName"
-  #   mountPath: "/opt/user/dir/mount"
-
-  # TODO 921 separate resource config, probably remains here
   rbac:
     # Specifies whether RBAC resources should be created
     create: true
 
-  # general properties
-  # TODO 921 deployments.controller.pod.container.livenessProbe
-  livenessProbe:
-    httpGet:
-      path: "/healthz"
-      port: 10254
-      scheme: HTTP
-    initialDelaySeconds: 5
-    timeoutSeconds: 5
-    periodSeconds: 10
-    successThreshold: 1
-    failureThreshold: 3
-  # TODO 921 deployments.controller.pod.container.readinessProbe
-  readinessProbe:
-    httpGet:
-      path: "/readyz"
-      port: 10254
-      scheme: HTTP
-    initialDelaySeconds: 5
-    timeoutSeconds: 5
-    periodSeconds: 10
-    successThreshold: 1
-    failureThreshold: 3
-  # TODO 921 deployments.controller.pod.container.resources
-  resources: {}
-  # Example reasonable setting for "resources":
-  # resources:
-  #   limits:
-  #     cpu: 100m
-  #     memory: 256Mi
-  #   requests:
-  #     cpu: 50m
-  #     memory: 128Mi
-
-  # TODO 921 app-level config, probably remains here
   konnect:
     enabled: false
 
@@ -1165,86 +982,6 @@ waitImage:
   # tag: 5
   pullPolicy: IfNotPresent
 
-
-
-# TODO 921 not moving these til we have a kong new-style section
-# they need different values for the controller and I want to preserve the existing defaults
-# for now
-# TODO 921 deployments.kong.pod.container.readinessProbe
-# readinessProbe for Kong pods
-readinessProbe:
-  httpGet:
-    path: "/status/ready"
-    port: status
-    scheme: HTTP
-  initialDelaySeconds: 5
-  timeoutSeconds: 5
-  periodSeconds: 10
-  successThreshold: 1
-  failureThreshold: 3
-
-# TODO 921 deployments.kong.pod.container.livenessProbe
-# livenessProbe for Kong pods
-livenessProbe:
-  httpGet:
-    path: "/status"
-    port: status
-    scheme: HTTP
-  initialDelaySeconds: 5
-  timeoutSeconds: 5
-  periodSeconds: 10
-  successThreshold: 1
-  failureThreshold: 3
-
-# TODO 921 deployments.kong.pod.container.startupProbe
-# startupProbe for Kong pods
-# startupProbe:
-#   httpGet:
-#     path: "/status"
-#     port: status
-#     scheme: HTTP
-#   initialDelaySeconds: 5
-#   timeoutSeconds: 5
-#   periodSeconds: 2
-#   successThreshold: 1
-#   failureThreshold: 40
-
-# TODO 921 deployments.kong.pod.container.lifecycle
-# Proxy container lifecycle hooks
-# Ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
-lifecycle:
-  preStop:
-    exec:
-      # kong quit has a default timeout of 10 seconds, and a default wait of 0 seconds.
-      # Note: together they should be less than the terminationGracePeriodSeconds setting below.
-      command:
-        - kong
-        - quit
-        - '--wait=15'
-
-
-
-
-
-# TODO 921 deployments.kong.pod.annotations
-# Annotation to be added to Kong pods
-podAnnotations:
-  kuma.io/gateway: enabled
-  traffic.sidecar.istio.io/includeInboundPorts: ""
-
-# TODO 921 deployments.kong.pod.labels
-# Labels to be added to Kong pods
-podLabels: {}
-
-# TODO 921 deployments.kong.replicaCount
-# Kong pod count.
-# It has no effect when autoscaling.enabled is set to true
-replicaCount: 1
-
-# TODO 921 deployments.kong.annotations
-# Annotations to be added to Kong deployment
-deploymentAnnotations: {}
-
 # TODO 921 this is a resource separate from the Deployment, but attached to it
 # not sure if we should nest under deployment config or create
 # `autoscaling.kong` and `autoscaling.controller` keys.
@@ -1307,38 +1044,6 @@ podSecurityPolicy:
     # If you use Kong Enterprise <1.5, this must be set to false.
     readOnlyRootFilesystem: true
 
-# TODO 921 .deployment.*.pod.priorityClassName
-priorityClassName: ""
-
-
-# TODO 921 deployments.*.pod.container.securityContext
-# securityContext for containers.
-containerSecurityContext:
-  readOnlyRootFilesystem: true
-  allowPrivilegeEscalation: false
-  runAsUser: 1000
-  runAsNonRoot: true
-  seccompProfile:
-    type: RuntimeDefault
-  capabilities:
-    drop:
-    - ALL
-
-# TODO 921 deployments.*.pod.dnsPolicy probably. confirm its actual location
-## Optional DNS configuration for Kong pods
-# dnsPolicy: ClusterFirst
-# dnsConfig:
-#   nameservers:
-#   - "10.100.0.10"
-#   options:
-#   - name: ndots
-#     value: "5"
-#   searches:
-#   - default.svc.cluster.local
-#   - svc.cluster.local
-#   - cluster.local
-#   - us-east-1.compute.internal
-
 # TODO 921 figure out if anything is necessary for this
 serviceMonitor:
   # Specifies whether ServiceMonitor for Prometheus operator should be created
@@ -1360,10 +1065,6 @@ serviceMonitor:
 # Kong Enterprise parameters
 # -----------------------------------------------------------------------------
 
-# TODO 921 the `enterprise` section is Kong-specific application-level
-# configurtion. As such it does not really map to any location in a Kubernetes
-# resource, so it probably just remains here at the root.
-
 # Toggle Kong Enterprise features on or off
 # RBAC and SMTP configuration have additional options that must all be set together
 # Other settings should be added to the "env" settings below
@@ -1609,3 +1310,7 @@ extraObjects: []
 #   config:
 #     per_consumer: false
 #   plugin: prometheus
+
+test:
+  # Enable creation of test resources for use with "helm test"
+  enabled: false

From 2c4570edf9b3706e41b0af1a83b06bcb2470ac75 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Fri, 10 Nov 2023 17:27:09 -0800
Subject: [PATCH 23/24] wip: template rendering fixed, need to fix volumes for
 install again

---
 charts/kong/templates/config-dbless.yaml      |  2 +-
 charts/kong/templates/controller-deploy.yaml  |  8 +++-
 .../kong/templates/deployment-controller.yaml |  4 +-
 charts/kong/templates/kong-deploy.yaml        | 21 ++++------
 .../templates/migrations-post-upgrade.yaml    | 40 +++++++++++--------
 .../templates/migrations-pre-upgrade.yaml     | 40 +++++++++++--------
 charts/kong/templates/migrations.yaml         | 40 +++++++++++--------
 charts/kong/values.yaml                       |  7 ++--
 8 files changed, 91 insertions(+), 71 deletions(-)

diff --git a/charts/kong/templates/config-dbless.yaml b/charts/kong/templates/config-dbless.yaml
index 5619b59a5..65c55c44f 100644
--- a/charts/kong/templates/config-dbless.yaml
+++ b/charts/kong/templates/config-dbless.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.deployment.kong.enabled }}
-{{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off")) }}
+{{- if (and (not .Values.ingressController.enabled) (eq .Values.deployment.kong.pod.container.env.database "off")) }}
 {{- if not (or .Values.dblessConfig.configMap .Values.dblessConfig.secret) }}
 {{- if .Values.dblessConfig.config }}
 apiVersion: v1
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index c0fbf32c2..f9396ee85 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -17,14 +17,20 @@
 # in ingress chart this is relying on different fullnames. can't simply format since there multiple keys,
 # need to pass a suffix to the helper
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.controllerSelectorLabels" .) -}}
+{{- $_ := set $deployPreRender "serviceAccount" .Values.serviceAccount -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
 # TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
 # pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
 # unnecessary volumes doesn't really hurt us
+{{/*
 {{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
 # TODO 921 this is not pulling from the new location
 {{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
+*/}}
+{{- $_ := set $deployPreRender "volumes" "" -}}
+# TODO 921 this is not pulling from the new location
+{{- $_ := set $deployPreRender "userVolumes" "" -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "component" "controller" -}}
@@ -41,7 +47,7 @@
 {{- $_ := set $containerPre "env" (include "kong.env.controller" .) -}}
 {{- $_ := set $containerPre "disableProbes" .Values.ingressController.disableProbes -}}
 {{- $_ := set $containerPre "adminApi" .Values.ingressController.adminApi -}}
-{{- $_ := set $containerPre "serviceAccount" .Values.deployment.controller.pod.serviceAccount -}}
+{{- $_ := set $containerPre "serviceAccount" .Values.serviceAccount -}}
 {{- $_ := set $containerPre "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $containerPre "serviceAccountName" (include "kong.serviceAccountName" .) -}}
 {{- $_ := set $containerConfig "pre" $containerPre -}}
diff --git a/charts/kong/templates/deployment-controller.yaml b/charts/kong/templates/deployment-controller.yaml
index cd2de4ae8..27a5b1e49 100644
--- a/charts/kong/templates/deployment-controller.yaml
+++ b/charts/kong/templates/deployment-controller.yaml
@@ -79,10 +79,10 @@ spec:
       {{- if .pod.priorityClassName }}
       priorityClassName: "{{ .pod.priorityClassName }}"
       {{- end }}
-      {{- if or .pod.serviceAccount.create .pod.serviceAccount.name }}
+      {{- if or .pre.deployment.serviceAccount.create .pre.deployment.serviceAccount.name }}
       serviceAccountName: {{ .pre.deployment.serviceAccountName }}
       {{- end }}
-      {{- if (and (or .pod.serviceAccount.create .pod.serviceAccount.name) .pod.serviceAccount.automountServiceAccountToken) }}
+      {{- if (and (or .pre.deployment.serviceAccount.create .pre.deployment.serviceAccount.name) .pre.deployment.serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
       {{- else }}
       automountServiceAccountToken: false
diff --git a/charts/kong/templates/kong-deploy.yaml b/charts/kong/templates/kong-deploy.yaml
index cf75f71ab..eb50ded37 100644
--- a/charts/kong/templates/kong-deploy.yaml
+++ b/charts/kong/templates/kong-deploy.yaml
@@ -23,16 +23,22 @@
 # need to pass a suffix to the helper
 {{- $_ := set $deployPreRender "selectorMatchLabels" (include "kong.selectorLabels" .) -}}
 # TODO 921 SA stuff can probably be refactored to be cleaner, but as-is it's functional
+{{- $_ := set $deployPreRender "serviceAccount" .Values.serviceAccount -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
-{{- $_ := set $deployPreRender "serviceAccount" .Values.serviceAccount -}}
 # TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
 # pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
 # unnecessary volumes doesn't really hurt us
+{{/*
 {{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
 {{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
 {{- $_ := set $containerPreRender "volumeMounts" (include "kong.volumeMounts" .) -}}
 {{- $_ := set $containerPreRender "userVolumeMounts" (include "kong.userDefinedVolumeMounts" .) -}}
+*/}}
+{{- $_ := set $deployPreRender "volumes" "" -}}
+{{- $_ := set $deployPreRender "userVolumes" "" -}}
+{{- $_ := set $containerPreRender "volumeMounts" "" -}}
+{{- $_ := set $containerPreRender "userVolumeMounts" (dict) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "component" "proxy" -}}
@@ -55,21 +61,10 @@
 {{- $_ := set $serviceConfig "clustertelemetry" .Values.clustertelemetry -}}
 {{- $_ := set $containerPreRender "svc" $serviceConfig -}}
 
-{{- $_ := set $containerPreRender "serviceAccount" .Values.deployment.kong.pod.serviceAccount -}}
+{{- $_ := set $containerPreRender "serviceAccount" .Values.serviceAccount -}}
 {{- $_ := set $containerPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $containerPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
 
-# TODO 921 there's some duplication here from pre.container, but that allows everything in env
-# that was previously at the root root (.Values) to remain at the new scoped root, rather than
-# having some things in . and some things in .envPre or whatever
-{{- $envRoot := dict -}}
-{{- $_ := set $envRoot "ingressControllerEnabled" .Values.deployment.controller.enabled -}}
-{{- $_ := set $envRoot "svc" $serviceConfig -}}
-{{- $_ := set $envRoot "certificates" .Values.certificates -}}
-{{- $_ := set $envRoot "enterprise" .Values.enterprise -}}
-{{- $_ := set $containerPreRender "envRoot" $envRoot -}}
-# TODO decide which strat to use, separate env or just shove everything into container
-{{- $envRoot := dict -}}
 {{- $_ := set $containerPreRender "ingressControllerEnabled" .Values.deployment.controller.enabled -}}
 {{- $_ := set $containerPreRender "Capabilities" .Capabilities -}}
 {{- $_ := set $containerPreRender "Release" .Release -}}
diff --git a/charts/kong/templates/migrations-post-upgrade.yaml b/charts/kong/templates/migrations-post-upgrade.yaml
index 6b1b38e32..44b408735 100644
--- a/charts/kong/templates/migrations-post-upgrade.yaml
+++ b/charts/kong/templates/migrations-post-upgrade.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.deployment.kong.enabled }}
-{{- if (and .Values.migrations.postUpgrade (not (eq .Values.env.database "off"))) }}
+{{- if (and .Values.migrations.postUpgrade (not (eq .Values.deployment.kong.pod.container.env.database "off"))) }}
 # Why is this Job duplicated and not using only helm hooks?
 # See: https://github.com/helm/charts/pull/7362
 apiVersion: batch/v1
@@ -29,29 +29,29 @@ spec:
       {{- range $key, $value := .Values.migrations.annotations }}
         {{ $key }}: {{ $value | quote }}
       {{- end }}
-      {{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+      {{- if (and (not .Values.deployment.kong.pod.automountServiceAccountToken) (or .Values.serviceAccount.create .Values.serviceAccount.name)) }}
         kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
       {{- end }}
       {{- end }}
     spec:
-      {{- if or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name }}
+      {{- if or .Values.serviceAccount.create .Values.serviceAccount.name }}
       serviceAccountName: {{ template "kong.serviceAccountName" . }}
       {{- end }}
-      {{- if (and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) .Values.deployment.serviceAccount.automountServiceAccountToken) }}
+      {{- if (and (or .Values.serviceAccount.create .Values.serviceAccount.name) .Values.serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
       {{- else }}
       automountServiceAccountToken: false
       {{ end }}
-      {{- if .Values.image.pullSecrets }}
+      {{- if .Values.deployment.kong.pod.container.image.pullSecrets }}
       imagePullSecrets:
-      {{- range .Values.image.pullSecrets }}
+      {{- range .Values.deployment.kong.pod.container.image.pullSecrets }}
         - name: {{ . }}
       {{- end }}
       {{- end }}
-      {{- if (or (and (.Values.postgresql.enabled) .Values.waitImage.enabled) .Values.deployment.initContainers) }}
+      {{- if (or (and (.Values.postgresql.enabled) .Values.waitImage.enabled) .Values.deployment.kong.pod.initContainers) }}
       initContainers:
-      {{- if .Values.deployment.initContainers }}
-      {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+      {{- if .Values.deployment.kong.pod.initContainers }}
+      {{- toYaml .Values.deployment.kong.pod.initContainers | nindent 6 }}
       {{- end }}
       {{- if (and (.Values.postgresql.enabled) .Values.waitImage.enabled) }}
       {{- include "kong.wait-for-postgres" . | nindent 6 }}
@@ -62,27 +62,33 @@ spec:
       {{- toYaml .Values.migrations.sidecarContainers | nindent 6 }}
       {{- end }}
       - name: {{ template "kong.name" . }}-post-upgrade-migrations
-        image: {{ include "kong.getRepoTag" .Values.image }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ include "kong.getRepoTag" .Values.deployment.kong.pod.container.image }}
+        imagePullPolicy: {{ .Values.deployment.kong.pod.container.image.pullPolicy }}
         securityContext:
-        {{ toYaml .Values.containerSecurityContext | nindent 10 }} 
+        {{ toYaml .Values.deployment.kong.pod.container.securityContext | nindent 10 }} 
+{{/* TODO 921 needs the same inputs as the main proxy container
         env:
+*/}}
         {{- include "kong.no_daemon_env" . | nindent 8 }}
         args: [ "kong", "migrations", "finish" ]
+{{/* TODO 921 volume stuff is a mess
         volumeMounts:
         {{- include "kong.volumeMounts" . | nindent 8 }}
-        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 8 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment.kong | nindent 8 }}
+*/}}
         resources:
         {{- toYaml .Values.migrations.resources | nindent 10 }}
+{{/* TODO 921 wat why was this not just a value dump
       securityContext:
       {{- include "kong.podsecuritycontext" . | nindent 8 }}
-      {{- if .Values.nodeSelector }}
+*/}}
+      {{- if .Values.deployment.kong.pod.nodeSelector }}
       nodeSelector:
-      {{- toYaml .Values.nodeSelector | nindent 8 }}
+      {{- toYaml .Values.deployment.kong.pod.nodeSelector | nindent 8 }}
       {{- end }}
-      {{- if .Values.tolerations }}
+      {{- if .Values.deployment.kong.pod.tolerations }}
       tolerations:
-      {{- toYaml .Values.tolerations | nindent 8 }}
+      {{- toYaml .Values.deployment.kong.pod.tolerations | nindent 8 }}
       {{- end }}
       restartPolicy: OnFailure
       volumes:
diff --git a/charts/kong/templates/migrations-pre-upgrade.yaml b/charts/kong/templates/migrations-pre-upgrade.yaml
index f5002aec9..0e6cd0831 100644
--- a/charts/kong/templates/migrations-pre-upgrade.yaml
+++ b/charts/kong/templates/migrations-pre-upgrade.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.deployment.kong.enabled }}
-{{- if (and .Values.migrations.preUpgrade (not (eq .Values.env.database "off"))) }}
+{{- if (and .Values.migrations.preUpgrade (not (eq .Values.deployment.kong.pod.container.env.database "off"))) }}
 # Why is this Job duplicated and not using only helm hooks?
 # See: https://github.com/helm/charts/pull/7362
 apiVersion: batch/v1
@@ -31,29 +31,29 @@ spec:
       {{- range $key, $value := .Values.migrations.annotations }}
         {{ $key }}: {{ $value | quote }}
       {{- end }}
-      {{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+      {{- if (and (not .Values.deployment.kong.pod.automountServiceAccountToken) (or .Values.serviceAccount.create .Values.serviceAccount.name)) }}
         kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
       {{- end }}
       {{- end }}
     spec:
-      {{- if or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name }}
+      {{- if or .Values.serviceAccount.create .Values.serviceAccount.name }}
       serviceAccountName: {{ template "kong.serviceAccountName" . }}
       {{- end }}
-      {{- if (and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) .Values.deployment.serviceAccount.automountServiceAccountToken) }}
+      {{- if (and (or .Values.serviceAccount.create .Values.serviceAccount.name) .Values.serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
       {{- else }}
       automountServiceAccountToken: false
       {{ end }}
-      {{- if .Values.image.pullSecrets }}
+      {{- if .Values.deployment.kong.pod.container.image.pullSecrets }}
       imagePullSecrets:
-      {{- range .Values.image.pullSecrets }}
+      {{- range .Values.deployment.kong.pod.container.image.pullSecrets }}
         - name: {{ . }}
       {{- end }}
       {{- end }}
-      {{- if (or (and (.Values.postgresql.enabled) .Values.waitImage.enabled) .Values.deployment.initContainers) }}
+      {{- if (or (and (.Values.postgresql.enabled) .Values.waitImage.enabled) .Values.deployment.kong.pod.initContainers) }}
       initContainers:
-      {{- if .Values.deployment.initContainers }}
-      {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+      {{- if .Values.deployment.kong.pod.initContainers }}
+      {{- toYaml .Values.deployment.kong.pod.initContainers | nindent 6 }}
       {{- end }}
       {{- if (and (.Values.postgresql.enabled) .Values.waitImage.enabled) }}
       {{- include "kong.wait-for-postgres" . | nindent 6 }}
@@ -64,27 +64,33 @@ spec:
       {{- toYaml .Values.migrations.sidecarContainers | nindent 6 }}
       {{- end }}
       - name: {{ template "kong.name" . }}-upgrade-migrations
-        image: {{ include "kong.getRepoTag" .Values.image }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ include "kong.getRepoTag" .Values.deployment.kong.pod.container.image }}
+        imagePullPolicy: {{ .Values.deployment.kong.pod.container.image.pullPolicy }}
         securityContext:
-        {{ toYaml .Values.containerSecurityContext | nindent 10 }}
+        {{ toYaml .Values.deployment.kong.pod.container.securityContext | nindent 10 }}
+{{/* TODO 921 needs the same inputs as the main proxy container
         env:
+*/}}
         {{- include "kong.no_daemon_env" . | nindent 8 }}
         args: [ "kong", "migrations", "up" ]
+{{/* TODO 921 volume stuff is a mess
         volumeMounts:
         {{- include "kong.volumeMounts" . | nindent 8 }}
-        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 8 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment.kong | nindent 8 }}
+*/}}
         resources:
         {{- toYaml .Values.migrations.resources| nindent 10 }}
+{{/* TODO 921 wat why was this not just a value dump
       securityContext:
       {{- include "kong.podsecuritycontext" . | nindent 8 }}
-      {{- if .Values.nodeSelector }}
+*/}}
+      {{- if .Values.deployment.kong.pod.nodeSelector }}
       nodeSelector:
-      {{- toYaml .Values.nodeSelector | nindent 8 }}
+      {{- toYaml .Values.deployment.kong.pod.nodeSelector | nindent 8 }}
       {{- end }}
-      {{- if .Values.tolerations }}
+      {{- if .Values.deployment.kong.pod.tolerations }}
       tolerations:
-      {{- toYaml .Values.tolerations | nindent 8 }}
+      {{- toYaml .Values.deployment.kong.pod.tolerations | nindent 8 }}
       {{- end }}
       restartPolicy: OnFailure
       volumes:
diff --git a/charts/kong/templates/migrations.yaml b/charts/kong/templates/migrations.yaml
index a996fcd13..e81d494fa 100644
--- a/charts/kong/templates/migrations.yaml
+++ b/charts/kong/templates/migrations.yaml
@@ -11,7 +11,7 @@
   {{- $runInit = .Values.migrations.init -}}
 {{- end -}}
 
-{{- if (and ($runInit) (not (eq .Values.env.database "off"))) }}
+{{- if (and ($runInit) (not (eq .Values.deployment.kong.pod.container.env.database "off"))) }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@@ -39,29 +39,29 @@ spec:
       {{- range $key, $value := .Values.migrations.annotations }}
         {{ $key }}: {{ $value | quote }}
       {{- end }}
-      {{- if (and (not .Values.deployment.serviceAccount.automountServiceAccountToken) (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name)) }}
+      {{- if (and (not .Values.deployment.kong.pod.automountServiceAccountToken) (or .Values.serviceAccount.create .Values.serviceAccount.name)) }}
         kuma.io/service-account-token-volume: {{ template "kong.serviceAccountTokenName" . }}
       {{- end }}
       {{- end }}
     spec:
-      {{- if or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name }}
+      {{- if or .Values.serviceAccount.create .Values.serviceAccount.name }}
       serviceAccountName: {{ template "kong.serviceAccountName" . }}
       {{- end }}
-      {{- if (and (or .Values.deployment.serviceAccount.create .Values.deployment.serviceAccount.name) .Values.deployment.serviceAccount.automountServiceAccountToken) }}
+      {{- if (and (or .Values.serviceAccount.create .Values.serviceAccount.name) .Values.serviceAccount.automountServiceAccountToken) }}
       automountServiceAccountToken: true
       {{- else }}
       automountServiceAccountToken: false
       {{ end }}
-      {{- if .Values.image.pullSecrets }}
+      {{- if .Values.deployment.kong.pod.container.image.pullSecrets }}
       imagePullSecrets:
-      {{- range .Values.image.pullSecrets }}
+      {{- range .Values.deployment.kong.pod.container.image.pullSecrets }}
         - name: {{ . }}
       {{- end }}
       {{- end }}
-      {{- if (or (and (.Values.postgresql.enabled) .Values.waitImage.enabled) .Values.deployment.initContainers) }}
+      {{- if (or (and (.Values.postgresql.enabled) .Values.waitImage.enabled) .Values.deployment.kong.pod.initContainers) }}
       initContainers:
-      {{- if .Values.deployment.initContainers }}
-      {{- toYaml .Values.deployment.initContainers | nindent 6 }}
+      {{- if .Values.deployment.kong.pod.initContainers }}
+      {{- toYaml .Values.deployment.kong.pod.initContainers | nindent 6 }}
       {{- end }}
       {{- if (and (.Values.postgresql.enabled) .Values.waitImage.enabled) }}
       {{- include "kong.wait-for-postgres" . | nindent 6 }}
@@ -72,27 +72,33 @@ spec:
       {{- toYaml .Values.migrations.sidecarContainers | nindent 6 }}
       {{- end }}
       - name: {{ template "kong.name" . }}-migrations
-        image: {{ include "kong.getRepoTag" .Values.image }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        image: {{ include "kong.getRepoTag" .Values.deployment.kong.pod.container.image }}
+        imagePullPolicy: {{ .Values.deployment.kong.pod.container.image.pullPolicy }}
         securityContext:
-        {{ toYaml .Values.containerSecurityContext | nindent 10 }} 
+        {{ toYaml .Values.deployment.kong.pod.container.securityContext | nindent 10 }} 
+{{/* TODO 921 needs the same inputs as the main proxy container
         env:
         {{- include "kong.no_daemon_env" . | nindent 8 }}
+*/}}
         args: [ "kong", "migrations", "bootstrap" ]
         volumeMounts:
+{{/* TODO 921 volume stuff is a mess
         {{- include "kong.volumeMounts" . | nindent 8 }}
-        {{- include "kong.userDefinedVolumeMounts" .Values.deployment | nindent 8 }}
+        {{- include "kong.userDefinedVolumeMounts" .Values.deployment.kong | nindent 8 }}
+*/}}
         resources:
         {{- toYaml .Values.migrations.resources | nindent 10 }}
+{{/* TODO 921 wat why was this not just a value dump
       securityContext:
       {{- include "kong.podsecuritycontext" . | nindent 8 }}
-      {{- if .Values.nodeSelector }}
+*/}}
+      {{- if .Values.deployment.kong.pod.nodeSelector }}
       nodeSelector:
-      {{- toYaml .Values.nodeSelector | nindent 8 }}
+      {{- toYaml .Values.deployment.kong.pod.nodeSelector | nindent 8 }}
       {{- end }}
-      {{- if .Values.tolerations }}
+      {{- if .Values.deployment.kong.pod.tolerations }}
       tolerations:
-      {{- toYaml .Values.tolerations | nindent 8 }}
+      {{- toYaml .Values.deployment.kong.pod.tolerations | nindent 8 }}
       {{- end }}
       restartPolicy: OnFailure
       volumes:
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 20305b5f5..6cb273f3d 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -211,9 +211,10 @@ deployment:
     # minReadySeconds: 60
     pod:
       labels: {}
-      annotations:
-        kuma.io/gateway: enabled
-        traffic.sidecar.istio.io/includeInboundPorts: ""
+      annotations: {}
+      # TODO 921 whoever decided that these needed to evaluate templates is the bane of my existence
+      # kuma.io/gateway: enabled
+      # traffic.sidecar.istio.io/includeInboundPorts: ""
       automountServiceAccountToken: false
       hostNetwork: false
       # Affinity for pod assignment

From dbdf2d25d82777e27220c9c9742ae72c25671a30 Mon Sep 17 00:00:00 2001
From: Travis Raines <571832+rainest@users.noreply.github.com>
Date: Tue, 14 Nov 2023 18:17:25 -0800
Subject: [PATCH 24/24] wip: volume fun and controller enabled

---
 charts/kong/templates/_helpers.tpl            | 291 +-----------------
 charts/kong/templates/admission-webhook.yaml  |  13 +-
 charts/kong/templates/config-dbless.yaml      |   2 +-
 charts/kong/templates/containers.yaml         |   7 +-
 charts/kong/templates/controller-deploy.yaml  |  15 +-
 .../templates/controller-rbac-resources.yaml  |   4 +-
 charts/kong/templates/env.yaml                |   4 +-
 charts/kong/templates/ingress-class.yaml      |   2 +-
 charts/kong/templates/kong-deploy.yaml        |  23 +-
 charts/kong/templates/servicemonitor.yaml     |   2 +-
 charts/kong/values.yaml                       |   6 +-
 11 files changed, 45 insertions(+), 324 deletions(-)

diff --git a/charts/kong/templates/_helpers.tpl b/charts/kong/templates/_helpers.tpl
index b721799d5..84e27d1df 100644
--- a/charts/kong/templates/_helpers.tpl
+++ b/charts/kong/templates/_helpers.tpl
@@ -390,8 +390,8 @@ Return the admin API service name for service discovery
     {{- $_ := required ".ingressController.gatewayDiscovery.adminApiService.name has to be provided when .Values.ingressController.gatewayDiscovery.enabled is set to true"  $adminApiServiceName -}}
   {{- end }}
 
-  {{- if (semverCompare "< 2.9.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
-  {{- fail (printf "Gateway discovery is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+  {{- if (semverCompare "< 2.9.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
+  {{- fail (printf "Gateway discovery is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
   {{- end }}
 
   {{- if .Values.deployment.kong.enabled }}
@@ -467,257 +467,6 @@ The name of the Service which will be used by the controller to update the Ingre
 {{- end -}}
 
 
-{{- define "kong.userDefinedVolumes" -}}
-{{- if .Values.deployment.userDefinedVolumes }}
-{{- toYaml .Values.deployment.userDefinedVolumes }}
-{{- end }}
-{{- end -}}
-
-{{- define "kong.volumes" -}}
-- name: {{ template "kong.fullname" . }}-prefix-dir
-  emptyDir:
-    sizeLimit: {{ .Values.deployment.kong.pod.container.prefixDir.sizeLimit }}
-- name: {{ template "kong.fullname" . }}-tmp
-  emptyDir:
-    sizeLimit: {{ .Values.deployment.kong.pod.container.tmpDir.sizeLimit }}
-{{- if (and (not .Values.serviceAccount.automountServiceAccountToken) (or .Values.serviceAccount.create .Values.serviceAccount.name)) }}
-- name: {{ template "kong.serviceAccountTokenName" . }}
-  {{- /* Due to GKE versions (e.g. v1.23.15-gke.1900) we need to handle pre-release part of the version as well.
-  See the related documentation of semver module that Helm depends on for semverCompare:
-  https://github.com/Masterminds/semver#working-with-prerelease-versions
-  Related Helm issue: https://github.com/helm/helm/issues/3810 */}}
-  {{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version }}
-  projected:
-    sources:
-    - serviceAccountToken:
-        expirationSeconds: 3607
-        path: token
-    - configMap:
-        items:
-        - key: ca.crt
-          path: ca.crt
-        name: kube-root-ca.crt
-    - downwardAPI:
-        items:
-        - fieldRef:
-            apiVersion: v1
-            fieldPath: metadata.namespace
-          path: namespace
-  {{- else }}
-  secret:
-    secretName: {{ template "kong.serviceAccountTokenName" . }}
-    items:
-    - key: token
-      path: token
-    - key: ca.crt
-      path: ca.crt
-    - key: namespace
-      path: namespace
-  {{- end }}
-{{- end }}
-{{- if and ( .Capabilities.APIVersions.Has "cert-manager.io/v1" ) .Values.certificates.enabled -}}
-{{- if .Values.certificates.cluster.enabled }}
-- name: {{ include "kong.fullname" . }}-cluster-cert
-  secret:
-    secretName: {{ include "kong.fullname" . }}-cluster-cert
-{{- end }}
-{{- if .Values.certificates.proxy.enabled }}
-- name: {{ include "kong.fullname" . }}-proxy-cert
-  secret:
-    secretName: {{ include "kong.fullname" . }}-proxy-cert
-{{- end }}
-{{- if .Values.certificates.admin.enabled }}
-- name: {{ include "kong.fullname" . }}-admin-cert
-  secret:
-    secretName: {{ include "kong.fullname" . }}-admin-cert
-{{- end }}
-{{- if .Values.enterprise.enabled }}
-{{- if .Values.certificates.portal.enabled }}
-- name: {{ include "kong.fullname" . }}-portal-cert
-  secret:
-    secretName: {{ include "kong.fullname" . }}-portal-cert
-{{- end }}
-{{- end }}
-{{- end }}
-{{- if (and (.Values.postgresql.enabled) .Values.waitImage.enabled) }}
-- name: {{ template "kong.fullname" . }}-bash-wait-for-postgres
-  configMap:
-    name: {{ template "kong.fullname" . }}-bash-wait-for-postgres
-    defaultMode: 0755
-{{- end }}
-{{- range .Values.plugins.configMaps }}
-- name: kong-plugin-{{ .pluginName }}
-  configMap:
-    name: {{ .name }}
-{{- range .subdirectories }}
-- name: {{ .name }}
-  configMap:
-    name: {{ .name }}
-{{- end }}
-{{- end }}
-{{- range .Values.plugins.secrets }}
-- name: kong-plugin-{{ .pluginName }}
-  secret:
-    secretName: {{ .name }}
-{{- range .subdirectories }}
-- name: {{ .name }}
-  secret:
-    secretName: {{ .name }}
-{{- end }}
-{{- end }}
-
-{{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off")) }}
-  {{- $dblessSourceCount := (add (.Values.dblessConfig.configMap | len | min 1) (.Values.dblessConfig.secret | len | min 1) (.Values.dblessConfig.config | len | min 1)) -}}
-  {{- if gt $dblessSourceCount 1 -}}
-    {{- fail "Ambiguous configuration: only one of of .Values.dblessConfig.configMap, .Values.dblessConfig.secret, and .Values.dblessConfig.config can be set." -}}
-  {{- else if eq $dblessSourceCount 1 }}
-- name: kong-custom-dbless-config-volume
-    {{- if .Values.dblessConfig.configMap }}
-  configMap:
-    name: {{ .Values.dblessConfig.configMap }}
-    {{- else if .Values.dblessConfig.secret }}
-  secret:
-    secretName: {{ .Values.dblessConfig.secret }}
-    {{- else }}
-  configMap:
-    name: {{ template "kong.dblessConfig.fullname" . }}
-    {{- end }}
-  {{- end }}
-{{- end }}
-
-{{- if and .Values.ingressController.enabled .Values.ingressController.admissionWebhook.enabled }}
-- name: webhook-cert
-  secret:
-    {{- if .Values.ingressController.admissionWebhook.certificate.provided }}
-    secretName: {{ .Values.ingressController.admissionWebhook.certificate.secretName }}
-    {{- else }}
-    secretName: {{ template "kong.fullname" . }}-validation-webhook-keypair
-    {{- end }}
-{{- end }}
-{{- if or $.Values.admin.tls.client.secretName $.Values.admin.tls.client.caBundle }}
-- name: admin-client-ca
-  configMap:
-    name: {{ template "kong.fullname" . }}-admin-client-ca
-{{- end -}}
-{{- range $secretVolume := .Values.secretVolumes }}
-- name: {{ . }}
-  secret:
-    secretName: {{ . }}
-{{- end }}
-{{- range .Values.extraConfigMaps }}
-- name: {{ .name }}
-  configMap:
-    name: {{ .name }}
-{{- end }}
-{{- range .Values.extraSecrets }}
-- name: {{ .name }}
-  secret:
-    secretName: {{ .name }}
-{{- end }}
-{{- if and .Values.ingressController.adminApi.tls.client.enabled .Values.ingressController.enabled }}
-- name: admin-api-cert
-  secret:
-    secretName: {{ template "adminApiService.certSecretName" . }}
-{{- end }}
-{{- end -}}
-
-{{- define "controller.adminApiCertVolumeMount" -}}
-{{- if .adminApi.tls.client.enabled }}
-- name: admin-api-cert
-  mountPath: /etc/secrets/admin-api-cert
-  readOnly: true
-{{- end -}}
-{{- end -}}
-
-{{- define "kong.userDefinedVolumeMounts" -}}
-{{- if .userDefinedVolumeMounts }}
-{{- toYaml .userDefinedVolumeMounts }}
-{{- end }}
-{{- end -}}
-
-{{- define "kong.volumeMounts" -}}
-- name: {{ template "kong.fullname" . }}-prefix-dir
-  mountPath: /kong_prefix/
-- name: {{ template "kong.fullname" . }}-tmp
-  mountPath: /tmp
-{{- if and ( .Capabilities.APIVersions.Has "cert-manager.io/v1" ) .Values.certificates.enabled -}}
-{{- if .Values.certificates.cluster.enabled }}
-- name: {{ include "kong.fullname" . }}-cluster-cert
-  mountPath: /etc/cert-manager/cluster/
-{{- end }}
-{{- if .Values.certificates.proxy.enabled }}
-- name: {{ include "kong.fullname" . }}-proxy-cert
-  mountPath: /etc/cert-manager/proxy/
-{{- end }}
-{{- if .Values.certificates.admin.enabled }}
-- name: {{ include "kong.fullname" . }}-admin-cert
-  mountPath: /etc/cert-manager/admin/
-{{- end }}
-{{- if .Values.enterprise.enabled }}
-{{- if .Values.certificates.portal.enabled }}
-- name: {{ include "kong.fullname" . }}-portal-cert
-  mountPath: /etc/cert-manager/portal/
-{{- end }}
-{{- end }}
-{{- end }}
-{{- $dblessSourceCount := (add (.Values.dblessConfig.configMap | len | min 1) (.Values.dblessConfig.secret | len | min 1) (.Values.dblessConfig.config | len | min 1)) -}}
-  {{- if eq $dblessSourceCount 1 -}}
-    {{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off")) }}
-- name: kong-custom-dbless-config-volume
-  mountPath: /kong_dbless/
-    {{- end }}
-  {{- end }}
-{{- if or $.Values.admin.tls.client.caBundle $.Values.admin.tls.client.secretName }}
-- name: admin-client-ca
-  mountPath: /etc/admin-client-ca/
-  readOnly: true
-{{- end -}}
-{{- range .Values.secretVolumes }}
-- name:  {{ . }}
-  mountPath: /etc/secrets/{{ . }}
-{{- end }}
-{{- range .Values.plugins.configMaps }}
-{{- $mountPath := printf "/opt/kong/plugins/%s" .pluginName }}
-- name:  kong-plugin-{{ .pluginName }}
-  mountPath: {{ $mountPath }}
-  readOnly: true
-{{- range .subdirectories }}
-- name: {{ .name  }}
-  mountPath: {{ printf "%s/%s" $mountPath ( .path | default .name ) }}
-  readOnly: true
-{{- end }}
-{{- end }}
-{{- range .Values.plugins.secrets }}
-{{- $mountPath := printf "/opt/kong/plugins/%s" .pluginName }}
-- name:  kong-plugin-{{ .pluginName }}
-  mountPath: {{ $mountPath }}
-  readOnly: true
-{{- range .subdirectories }}
-- name: {{ .name }}
-  mountPath: {{ printf "%s/%s" $mountPath .path }}
-  readOnly: true
-{{- end }}
-{{- end }}
-
-{{- range .Values.extraConfigMaps }}
-- name:  {{ .name }}
-  mountPath: {{ .mountPath }}
-
-  {{- if .subPath }}
-  subPath: {{ .subPath }}
-  {{- end }}
-{{- end }}
-{{- range .Values.extraSecrets }}
-- name:  {{ .name }}
-  mountPath: {{ .mountPath }}
-
-  {{- if .subPath }}
-  subPath: {{ .subPath }}
-  {{- end }}
-{{- end }}
-
-{{- end -}}
-
 {{/* TODO 921 this gets used for any Kong env. the migrations and init paths here don't yet
      build a dict, and so won't work without the removed .Values */}}
 {{- define "kong.plugins" -}}
@@ -786,7 +535,7 @@ The name of the Service which will be used by the controller to update the Ingre
     containerPort: {{ .Values.ingressController.admissionWebhook.port }}
     protocol: TCP
   {{- end }}
-  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .Values.ingressController.image)) -}}
+  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) -}}
   - name: cmetrics
     containerPort: 10255
     protocol: TCP
@@ -803,7 +552,7 @@ The name of the Service which will be used by the controller to update the Ingre
         apiVersion: v1
         fieldPath: metadata.namespace
 {{- include "kong.ingressController.env" .  | indent 2 }}
-  image: {{ include "kong.getRepoTag" .Values.ingressController.image }}
+  image: {{ include "kong.getRepoTag" .Values.deployment.controller.pod.container.image }}
   imagePullPolicy: {{ .Values.image.pullPolicy }}
 {{/* disableReadiness is a hidden setting to drop this block entirely for use with a debugger
      Helm value interpretation doesn't let you replace the default HTTP checks with any other
@@ -868,7 +617,7 @@ the template that it itself is using form the above sections.
 {{- $_ := set $autoEnv "KONG_ADMIN_ERROR_LOG" "/dev/stderr" -}}
 {{- $_ := set $autoEnv "KONG_STATUS_ERROR_LOG" "/dev/stderr" -}}
 
-{{- if .Values.ingressController.enabled -}}
+{{- if .Values.deployment.controller.enabled -}}
   {{- $_ := set $autoEnv "KONG_KIC" "on" -}}
 {{- end -}}
 
@@ -1044,7 +793,7 @@ the template that it itself is using form the above sections.
   {{- $_ := set $autoEnv "KONG_PG_PORT" "5432" }}
 {{- end }}
 
-{{- if (and (not .Values.ingressController.enabled) (eq .Values.env.database "off")) }}
+{{- if (and (not .Values.deployment.controller.enabled) (eq .Values.env.database "off")) }}
 {{- $dblessSourceCount := (add (.Values.dblessConfig.configMap | len | min 1) (.Values.dblessConfig.secret | len | min 1) (.Values.dblessConfig.config | len | min 1)) -}}
 {{- if eq $dblessSourceCount 1 -}}
   {{- $_ := set $autoEnv "KONG_DECLARATIVE_CONFIG" "/kong_dbless/kong.yml" -}}
@@ -1140,7 +889,7 @@ role sets used in the charts. Updating these requires separating out cluster
 resource roles into their separate templates.
 */}}
 {{- define "kong.kubernetesRBACRules" -}}
-{{- if (semverCompare ">= 3.0.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 3.0.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
 - apiGroups:
   - configuration.konghq.com
   resources:
@@ -1158,7 +907,7 @@ resource roles into their separate templates.
   - patch
   - update
 {{- end }}
-{{- if (semverCompare ">= 2.11.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 2.11.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
 - apiGroups:
   - configuration.konghq.com
   resources:
@@ -1176,7 +925,7 @@ resource roles into their separate templates.
   - patch
   - update
 {{- end }}
-{{- if (semverCompare "< 2.10.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare "< 2.10.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
 - apiGroups:
   - ""
   resources:
@@ -1442,24 +1191,6 @@ resource roles into their separate templates.
   - patch
   - update
 {{- end }}
-{{- if (.Capabilities.APIVersions.Has "networking.internal.knative.dev/v1alpha1") }}
-- apiGroups:
-  - networking.internal.knative.dev
-  resources:
-  - ingresses
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - networking.internal.knative.dev
-  resources:
-  - ingresses/status
-  verbs:
-  - get
-  - patch
-  - update
-{{- end }}
 - apiGroups:
   - networking.k8s.io
   resources:
@@ -1508,7 +1239,7 @@ Kubernetes Cluster-scoped resources it uses to build Kong configuration.
   - get
   - patch
   - update
-{{- if (semverCompare ">= 2.10.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 2.10.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
 - apiGroups:
   - apiextensions.k8s.io
   resources:
@@ -1582,7 +1313,7 @@ extensions/v1beta1
 
 {{- define "kong.proxy.compatibleReadiness" -}}
 {{- $proxyReadiness := .Values.readinessProbe -}}
-{{- if (or (semverCompare "< 3.3.0" (include "kong.effectiveVersion" .Values.image)) (and .Values.ingressController.enabled (semverCompare "< 2.11.0" (include "kong.effectiveVersion" .Values.ingressController.image)))) -}}
+{{- if (or (semverCompare "< 3.3.0" (include "kong.effectiveVersion" .Values.image)) (and .Values.deployment.controller.enabled (semverCompare "< 2.11.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)))) -}}
     {{- if (eq $proxyReadiness.httpGet.path "/status/ready") -}}
         {{- $_ := set $proxyReadiness.httpGet "path" "/status" -}}
     {{- end -}}
diff --git a/charts/kong/templates/admission-webhook.yaml b/charts/kong/templates/admission-webhook.yaml
index 8e223f406..6c9efac8e 100644
--- a/charts/kong/templates/admission-webhook.yaml
+++ b/charts/kong/templates/admission-webhook.yaml
@@ -1,4 +1,5 @@
-{{- if (and .Values.ingressController.admissionWebhook.enabled .Values.ingressController.enabled) }}
+# DEMONS: {{ (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image) }}
+{{- if (and .Values.admissionWebhook.enabled .Values.deployment.controller.enabled) }}
 {{- $certCert := "" -}}
 {{- $certKey := "" -}}
 {{- $caCert := "" -}}
@@ -69,10 +70,10 @@ webhooks:
     resources:
     - kongconsumers
     - kongplugins
-{{- if (semverCompare ">= 2.0.4" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 2.0.4" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
     - kongclusterplugins
 {{- end }}
-{{- if (semverCompare ">= 2.8.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 2.8.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
     - kongingresses
 {{- end }}
   - apiGroups:
@@ -80,16 +81,16 @@ webhooks:
     apiVersions:
     - 'v1'
     operations:
-{{- if (semverCompare ">= 2.12.1" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 2.12.1" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
     - CREATE
 {{- end }}
     - UPDATE
     resources:
     - secrets
-{{- if (semverCompare ">= 3.0.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 3.0.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
     - services
 {{- end }}
-{{- if (semverCompare ">= 2.12.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare ">= 2.12.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
   - apiGroups:
     - networking.k8s.io
     apiVersions:
diff --git a/charts/kong/templates/config-dbless.yaml b/charts/kong/templates/config-dbless.yaml
index 65c55c44f..9cc5af35e 100644
--- a/charts/kong/templates/config-dbless.yaml
+++ b/charts/kong/templates/config-dbless.yaml
@@ -1,5 +1,5 @@
 {{- if .Values.deployment.kong.enabled }}
-{{- if (and (not .Values.ingressController.enabled) (eq .Values.deployment.kong.pod.container.env.database "off")) }}
+{{- if (and (not .Values.deployment.controller.enabled) (eq .Values.deployment.kong.pod.container.env.database "off")) }}
 {{- if not (or .Values.dblessConfig.configMap .Values.dblessConfig.secret) }}
 {{- if .Values.dblessConfig.config }}
 apiVersion: v1
diff --git a/charts/kong/templates/containers.yaml b/charts/kong/templates/containers.yaml
index 2458a27eb..29bba15cb 100644
--- a/charts/kong/templates/containers.yaml
+++ b/charts/kong/templates/containers.yaml
@@ -141,7 +141,9 @@
   {{- end }}
   volumeMounts:
   {{- .pre.volumeMounts | nindent 4 }}
-  {{- .userDefinedVolumeMounts | toYaml | nindent 4 }}
+  {{- if .userDefinedVolumeMounts  -}}
+  {{ .userDefinedVolumeMounts | toYaml | nindent 4 }}
+  {{- end }}
   readinessProbe:
 {{ toYaml .readinessProbe | indent 4 }}
 {{/* TODO 921 this requires controller-related keys we aren't passing in yet
@@ -221,7 +223,6 @@
     readOnly: true
 {{- end }}
   {{- include "kong.userDefinedVolumeMounts" . | nindent 2 }}
-  # TODO 921 relies on some content under .ingressController
-  {{- include "controller.adminApiCertVolumeMount" .pre | nindent 2 }}
+  {{- include "kong.volumeMounts.controller.adminApiCertVolumeMount" . | nindent 2 }}
 {{- end -}}
 
diff --git a/charts/kong/templates/controller-deploy.yaml b/charts/kong/templates/controller-deploy.yaml
index f9396ee85..a7017d97e 100644
--- a/charts/kong/templates/controller-deploy.yaml
+++ b/charts/kong/templates/controller-deploy.yaml
@@ -20,17 +20,9 @@
 {{- $_ := set $deployPreRender "serviceAccount" .Values.serviceAccount -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
-# TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
-# pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
-# unnecessary volumes doesn't really hurt us
-{{/*
-{{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
-# TODO 921 this is not pulling from the new location
-{{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
-*/}}
-{{- $_ := set $deployPreRender "volumes" "" -}}
-# TODO 921 this is not pulling from the new location
-{{- $_ := set $deployPreRender "userVolumes" "" -}}
+
+{{- $_ := set $deployPreRender "volumes" (include "kong.volumes.controller" .) -}}
+{{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .Values.deployment.controller ) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "component" "controller" -}}
@@ -43,7 +35,6 @@
 {{- $containerConfig := .Values.deployment.controller.pod.container -}}
 {{- $containerPre := dict -}}
 {{- $_ := set $containerPre "admissionWebhook" .Values.admissionWebhook -}}
-# TODO 921 still global scoped
 {{- $_ := set $containerPre "env" (include "kong.env.controller" .) -}}
 {{- $_ := set $containerPre "disableProbes" .Values.ingressController.disableProbes -}}
 {{- $_ := set $containerPre "adminApi" .Values.ingressController.adminApi -}}
diff --git a/charts/kong/templates/controller-rbac-resources.yaml b/charts/kong/templates/controller-rbac-resources.yaml
index f5873f052..56194e3b6 100644
--- a/charts/kong/templates/controller-rbac-resources.yaml
+++ b/charts/kong/templates/controller-rbac-resources.yaml
@@ -1,4 +1,4 @@
-{{- if and .Values.ingressController.rbac.create .Values.ingressController.enabled -}}
+{{- if and .Values.ingressController.rbac.create .Values.deployment.controller.enabled -}}
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -35,7 +35,7 @@ rules:
       - configmaps
     verbs:
       - create
-{{- if (semverCompare "< 2.10.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+{{- if (semverCompare "< 2.10.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
   - apiGroups:
       - ""
     resources:
diff --git a/charts/kong/templates/env.yaml b/charts/kong/templates/env.yaml
index a2f3b82a1..973206c4a 100644
--- a/charts/kong/templates/env.yaml
+++ b/charts/kong/templates/env.yaml
@@ -325,8 +325,8 @@ the template that it itself is using form the above sections.
 */}}
 
 {{- if .Values.ingressController.konnect.enabled }}
-  {{- if (semverCompare "< 2.9.0" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
-  {{- fail (printf "Konnect sync is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.ingressController.image)) }}
+  {{- if (semverCompare "< 2.9.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
+  {{- fail (printf "Konnect sync is available in controller versions 2.9 and up. Detected %s" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) }}
   {{- end }}
 
   # TODO 921 removed a "must set GW discovery" check here since the setting will go away. dunno if we bother with a check on admin URL setting
diff --git a/charts/kong/templates/ingress-class.yaml b/charts/kong/templates/ingress-class.yaml
index d2ac47d69..4c5b5135a 100644
--- a/charts/kong/templates/ingress-class.yaml
+++ b/charts/kong/templates/ingress-class.yaml
@@ -1,6 +1,6 @@
 {{/* Default to not managing if unsupported or created outside this chart */}}
 {{- $includeIngressClass := false -}}
-{{- if .Values.ingressController.enabled -}}
+{{- if .Values.deployment.controller.enabled -}}
   {{- if (.Capabilities.APIVersions.Has "networking.k8s.io/v1/IngressClass") -}}
     {{- with (lookup "networking.k8s.io/v1" "IngressClass" "" .Values.ingressController.ingressClass) -}}
       {{- if (hasKey .metadata "annotations") -}}
diff --git a/charts/kong/templates/kong-deploy.yaml b/charts/kong/templates/kong-deploy.yaml
index eb50ded37..40e819ca4 100644
--- a/charts/kong/templates/kong-deploy.yaml
+++ b/charts/kong/templates/kong-deploy.yaml
@@ -26,19 +26,11 @@
 {{- $_ := set $deployPreRender "serviceAccount" .Values.serviceAccount -}}
 {{- $_ := set $deployPreRender "serviceAccountTokenName" (include "kong.serviceAccountTokenName" .) -}}
 {{- $_ := set $deployPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
-# TODO 921 this helper needs to be split somehow. as-is it includes things that are only useful for one
-# pod or the other, e.g. the controller does not need the prefix dir. as a placeholder, including
-# unnecessary volumes doesn't really hurt us
-{{/*
-{{- $_ := set $deployPreRender "volumes" (include "kong.volumes" .) -}}
-{{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .) -}}
-{{- $_ := set $containerPreRender "volumeMounts" (include "kong.volumeMounts" .) -}}
-{{- $_ := set $containerPreRender "userVolumeMounts" (include "kong.userDefinedVolumeMounts" .) -}}
-*/}}
-{{- $_ := set $deployPreRender "volumes" "" -}}
-{{- $_ := set $deployPreRender "userVolumes" "" -}}
-{{- $_ := set $containerPreRender "volumeMounts" "" -}}
-{{- $_ := set $containerPreRender "userVolumeMounts" (dict) -}}
+
+{{- $_ := set $deployPreRender "volumes" (include "kong.volumes.proxy" .) -}}
+{{- $_ := set $deployPreRender "userVolumes" (include "kong.userDefinedVolumes" .Values.deployment.kong ) -}}
+{{- $_ := set $containerPreRender "volumeMounts" (include "kong.volumeMounts.proxy" .) -}}
+{{- $_ := set $containerPreRender "userVolumeMounts" (include "kong.userDefinedVolumeMounts" .Values.deployment.kong ) -}}
 
 # TODO 921 ditto the deploy labels todo
 {{- $_ := set $podPreRender "component" "proxy" -}}
@@ -66,10 +58,14 @@
 {{- $_ := set $containerPreRender "serviceAccountName" (include "kong.serviceAccountName" .) -}}
 
 {{- $_ := set $containerPreRender "ingressControllerEnabled" .Values.deployment.controller.enabled -}}
+# TODO 921 Capabilities and Release got stuck here because it was the first place I used them, but
+# it's probably not a great home for them--they are not specific to the container and also get used
+# in the deployment-level volumes
 {{- $_ := set $containerPreRender "Capabilities" .Capabilities -}}
 {{- $_ := set $containerPreRender "Release" .Release -}}
 {{- $_ := set $containerPreRender "dblessConfig" .Values.dblessConfig -}}
 {{- $_ := set $containerPreRender "plugins" .Values.plugins -}}
+{{- $_ := set $containerPreRender "certificates" .Values.certificates -}}
 # TODO 921 we should probably take this opportunity to drop the postgresql subchart. keeping for
 # now to try and reduce scope (hah)
 {{- $_ := set $containerPreRender "postgresql" .Values.postgresql -}}
@@ -112,6 +108,7 @@
 {{- $volumeMounts := list -}}
 {{- $volumeMounts = mustAppend $volumeMounts (dict "foo" "bar") -}}
 
+# TODO 921 this is inconsistent with the others
 {{- $_ := set $containerConfig "pre" $containerPreRender -}}
 
 {{- $_ := set $podPreRender "container" (include "kong.proxy-container-new" $containerConfig ) -}}
diff --git a/charts/kong/templates/servicemonitor.yaml b/charts/kong/templates/servicemonitor.yaml
index b0f8b4d3a..72e965912 100644
--- a/charts/kong/templates/servicemonitor.yaml
+++ b/charts/kong/templates/servicemonitor.yaml
@@ -24,7 +24,7 @@ spec:
     {{- if .Values.serviceMonitor.metricRelabelings }}
     metricRelabelings: {{ toYaml .Values.serviceMonitor.metricRelabelings | nindent 6 }}
     {{- end }}
-  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .Values.ingressController.image)) -}}
+  {{ if (semverCompare ">= 2.0.0" (include "kong.effectiveVersion" .Values.deployment.controller.pod.container.image)) -}}
   - targetPort: cmetrics
     scheme: http
     {{- if .Values.serviceMonitor.interval }}
diff --git a/charts/kong/values.yaml b/charts/kong/values.yaml
index 6cb273f3d..b64b3ea34 100644
--- a/charts/kong/values.yaml
+++ b/charts/kong/values.yaml
@@ -327,9 +327,9 @@ deployment:
           capabilities:
             drop:
             - ALL
-        userDefinedVolumeMounts:
-        - name: webhook-cert
-          mountPath: /example
+        userDefinedVolumeMounts: {}
+        # - name: example
+        #   mountPath: /example
         # If you want to specify resources, uncomment the following
         # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
         resources: {}