-
Notifications
You must be signed in to change notification settings - Fork 83
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
add privateCA support for containerd images (#2079)
* draft commit for containerd ca support * base template config for containerd certs update * fix ci failure * update templates * file naming typo fix * update containerd config toml values * update containerd certificate ca script * add custom logic to check config_path * fix pre-commit * config update * disable docker certs update when containerd is enabled * update config values and template for flag change --------- Co-authored-by: Rishabh Karajgi <rishabh.karajgi@gmail.com>
- Loading branch information
1 parent
4c436df
commit b143df4
Showing
6 changed files
with
295 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
95 changes: 95 additions & 0 deletions
95
templates/trust-private-ca-on-all-nodes/containerd-ca-update-script.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| ################################ | ||
| ## Containerd toml ConfigMap ## | ||
| ################################ | ||
| {{- if and .Values.global.privateCaCertsAddToHost.enabled .Values.global.privateCaCertsAddToHost.addToContainerd }} | ||
| apiVersion: v1 | ||
| kind: ConfigMap | ||
| metadata: | ||
| name: {{ .Release.Name }}-cert-copy-and-toml-update | ||
| labels: | ||
| component: houston | ||
| tier: astronomer | ||
| release: {{ .Release.Name }} | ||
| chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" | ||
| heritage: {{ .Release.Service }} | ||
| data: | ||
| update-containerd-certs.sh: | | ||
| #!/usr/bin/env sh | ||
| if [ ! -f /hostcontainerd/config.toml ]; then | ||
| echo "No /hostcontainerd/config.toml found, exiting. Is this a containerd node?"; | ||
| exit 1; | ||
| fi | ||
| # backup the original ca certificates it is not already backed up | ||
| if [ ! -f /hostcontainerd/config.toml.bak ]; then | ||
| # backup /hostcontainerd/config.toml | ||
| cp /hostcontainerd/config.toml /hostcontainerd/config.toml.bak | ||
| fi | ||
| # ensure hsotcontainerd/certs.d exists | ||
| if [ ! -d /hostcontainerd/certs.d ]; then | ||
| mkdir -p /hostcontainerd/certs.d; | ||
| fi | ||
| # check if config_path exists | ||
| echo "checking if config_path key exists in /hostcontainerd/config.toml" | ||
| export config_key="config_path" | ||
| grep -w "$config_key" "/hostcontainerd/config.toml" | ||
| if [ $? -eq 0 ]; then | ||
| echo "The key '$config_key' exists in the configuration file." | ||
| export CONFIG_KEY=true | ||
| cat << EOF > /hostcontainerd/host.toml | ||
| {{ include "containerd.configToml" . | indent 4 }} | ||
| EOF | ||
| else | ||
| export CONFIG_KEY=false | ||
| fi | ||
|
|
||
| last_checksum=""; | ||
|
|
||
| while true; do | ||
| cp /hostcontainerd/config.toml.bak /hostcontainerd/working; | ||
| # loop through all /hostcontainerd/cert.d/*/*.crt using directory name as REGISTRY_HOST | ||
| for dir in /private-ca-certs/*; do | ||
| REGISTRY_HOST=$(basename $dir); | ||
| if [ ! -d /hostcontainerd/certs.d/$REGISTRY_HOST ]; then | ||
| mkdir -p /hostcontainerd/certs.d/$REGISTRY_HOST; | ||
| fi | ||
| cp $dir/*.crt /hostcontainerd/certs.d/$REGISTRY_HOST/; | ||
| cp $dir/*.crt /hostcontainerd/certs.d/registry.{{ .Values.global.baseDomain }}/; | ||
| if [ "$CONFIG_KEY" = true ]; then | ||
| cp /hostcontainerd/host.toml /hostcontainerd/certs.d/$REGISTRY_HOST/host.toml | ||
| else | ||
| cat << EOF >> /hostcontainerd/working | ||
| {{ include "containerd.configToml" . | indent 4 }} | ||
| EOF | ||
| fi | ||
| done | ||
|
|
||
| if [ "$CONFIG_KEY" = true ]; then | ||
| current_host_checksum=$(md5sum /hostcontainerd/host.toml | awk '{print $1}'); | ||
| if [ "$current_host_checksum" != "$last_host_checksum" ]; then | ||
| cp /hostcontainerd/host.toml /hostcontainerd/certs.d/$REGISTRY_HOST/host.toml; | ||
| echo "Updated /hostcontainerd/certs.d/$REGISTRY_HOST/host.toml"; | ||
| last_host_checksum="$current_host_checksum"; | ||
| echo "Restarting containerd on node"; | ||
| nsenter --target 1 --mount --uts --ipc --net --pid systemctl restart containerd | ||
| else | ||
| echo "no change in config sleep for 1 second"; | ||
| sleep 1; | ||
| fi | ||
| else | ||
| current_checksum=$(md5sum /hostcontainerd/working | awk '{print $1}'); | ||
| if [ "$current_checksum" != "$last_checksum" ]; then | ||
| cp /hostcontainerd/working /hostcontainerd/config.toml; | ||
| echo "Updated /hostcontainerd/config.toml"; | ||
| last_checksum="$current_checksum"; | ||
| echo "Restarting containerd on node"; | ||
| nsenter --target 1 --mount --uts --ipc --net --pid systemctl restart containerd | ||
| else | ||
| echo "no change in config sleep for 1 second"; | ||
| sleep 1; | ||
| fi | ||
| fi | ||
| done | ||
| {{- end }} |
93 changes: 93 additions & 0 deletions
93
templates/trust-private-ca-on-all-nodes/containerd-daemonset.yaml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| {{- if and .Values.global.privateCaCertsAddToHost.enabled .Values.global.privateCaCertsAddToHost.addToContainerd }} | ||
| ################################ | ||
| ## DaemonSet to mount the private root CA for containerd | ||
| ## | ||
| ## This can be used by enterprise with private | ||
| ## CAs that do not already install their root | ||
| ## certificate on the kubernetes nodes. | ||
| ################################# | ||
| apiVersion: apps/v1 | ||
| kind: DaemonSet | ||
| metadata: | ||
| name: {{ .Release.Name }}-containerd-ca-update | ||
| labels: | ||
| tier: platform | ||
| component: containerd-private-ca | ||
| release: {{ .Release.Name }} | ||
| spec: | ||
| # allow update to occur all at once | ||
| updateStrategy: | ||
| type: RollingUpdate | ||
| rollingUpdate: | ||
| maxUnavailable: "100%" | ||
| selector: | ||
| matchLabels: | ||
| tier: platform | ||
| component: containerd-private-ca | ||
| release: {{ .Release.Name }} | ||
| template: | ||
| metadata: | ||
| labels: | ||
| tier: platform | ||
| component: containerd-private-ca | ||
| release: {{ .Release.Name }} | ||
| annotations: | ||
| checksum/configmap: {{ include (print $.Template.BasePath "/trust-private-ca-on-all-nodes/containerd-ca-update-script.yaml") . | sha256sum }} | ||
| spec: | ||
| {{- if .Values.global.privateCaCertsAddToHost.containerdnodeAffinitys }} | ||
| affinity: | ||
| {{ toYaml .Values.global.privateCaCertsAddToHost.containerdnodeAffinitys | indent 8 }} | ||
| {{ end }} | ||
| containers: | ||
| - name: cert-copy-and-toml-update | ||
| image: {{ .Values.global.privateCaCertsAddToHost.certCopier.repository }}:{{ .Values.global.privateCaCertsAddToHost.certCopier.tag }} | ||
| command: | ||
| - "sh" | ||
| - "-c" | ||
| args: | ||
| - sh /cert-copy-and-toml-update.sh | ||
| imagePullPolicy: IfNotPresent | ||
| securityContext: | ||
| runAsUser: 0 | ||
| privileged: true | ||
| resources: | ||
| requests: | ||
| cpu: 1m | ||
| memory: "25Mi" | ||
| limits: | ||
| cpu: 50m | ||
| memory: "50Mi" | ||
| volumeMounts: | ||
| - name: hostcerts | ||
| mountPath: /host-trust-store | ||
| - mountPath: /hostcontainerd | ||
| name: hostcontainerd | ||
| readOnly: false | ||
| - mountPath: /cert-copy-and-toml-update.sh | ||
| name: cert-copy-and-toml-update | ||
| subPath: update-containerd-certs.sh | ||
| {{ range $secret_name := (.Values.global.privateCaCerts) }} | ||
| - name: {{ $secret_name }} | ||
| mountPath: /private-ca-certs/{{ $secret_name }}/{{ $secret_name }}.crt | ||
| subPath: {{ $secret_name }}.crt | ||
| {{- end }} | ||
| terminationGracePeriodSeconds: 1 | ||
| hostNetwork: true | ||
| hostPID: true | ||
| volumes: | ||
| - hostPath: | ||
| path: /etc/containerd | ||
| type: "" | ||
| name: hostcontainerd | ||
| - name: hostcerts | ||
| hostPath: | ||
| path: {{ .Values.global.privateCaCertsAddToHost.containerdCertConfigPath }}/registry.{{ .Values.global.baseDomain }}/ | ||
| - name: cert-copy-and-toml-update | ||
| configMap: | ||
| name: {{ .Release.Name }}-cert-copy-and-toml-update | ||
| {{ range $secret_name := (.Values.global.privateCaCerts) }} | ||
| - name: {{ $secret_name }} | ||
| secret: | ||
| secretName: {{ $secret_name }} | ||
| {{- end }} | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,97 @@ | ||
| from tests.chart_tests.helm_template_generator import render_chart | ||
| import pytest | ||
| from tests import supported_k8s_versions | ||
| from subprocess import CalledProcessError | ||
|
|
||
|
|
||
| @pytest.mark.parametrize( | ||
| "kube_version", | ||
| supported_k8s_versions, | ||
| ) | ||
| class TestContainerdPrivateCaDaemonset: | ||
| show_only = [ | ||
| "templates/trust-private-ca-on-all-nodes/containerd-daemonset.yaml", | ||
| "templates/trust-private-ca-on-all-nodes/containerd-ca-update-script.yaml", | ||
| ] | ||
|
|
||
| @staticmethod | ||
| def common_tests_daemonset(doc): | ||
| """Test things common to all daemonsets.""" | ||
| assert doc["kind"] == "DaemonSet" | ||
| assert doc["metadata"]["name"] == "release-name-containerd-ca-update" | ||
| assert ( | ||
| doc["spec"]["template"]["spec"]["containers"][0]["name"] | ||
| == "cert-copy-and-toml-update" | ||
| ) | ||
|
|
||
| def test_privateca_daemonset_disabled(self, kube_version): | ||
| """Test that no daemonset is rendered when privateCaCertsAddToHost is | ||
| disabled.""" | ||
| with pytest.raises(CalledProcessError): | ||
| render_chart( | ||
| kube_version=kube_version, | ||
| show_only=self.show_only, | ||
| values={ | ||
| "global": { | ||
| "privateCaCerts": [ | ||
| "private-ca-cert-foo", | ||
| "private-ca-cert-bar", | ||
| ], | ||
| "privateCaCertsAddToHost": { | ||
| "enabled": False, | ||
| "addToContainerd": False, | ||
| }, | ||
| } | ||
| }, | ||
| ) | ||
|
|
||
| def test_containerd_privateca_daemonset_enabled(self, kube_version): | ||
| """Test that the daemonset is rendered with valid properties when | ||
| enabled.""" | ||
| docs = render_chart( | ||
| kube_version=kube_version, | ||
| show_only=self.show_only, | ||
| values={ | ||
| "global": { | ||
| "privateCaCerts": ["private-ca-cert-foo", "private-ca-cert-bar"], | ||
| "privateCaCertsAddToHost": { | ||
| "enabled": True, | ||
| "addToContainerd": True, | ||
| }, | ||
| } | ||
| }, | ||
| ) | ||
|
|
||
| assert len(docs) == 2 | ||
| self.common_tests_daemonset(docs[0]) | ||
| assert len(docs[0]["spec"]["template"]["spec"]["containers"]) == 1 | ||
| cert_copier = docs[0]["spec"]["template"]["spec"]["containers"][0] | ||
| cert_copier["image"].startswith("alpine:3") | ||
|
|
||
| volmounts = cert_copier["volumeMounts"] | ||
|
|
||
| volmounts_expected = [ | ||
| {"name": "hostcerts", "mountPath": "/host-trust-store"}, | ||
| { | ||
| "mountPath": "/hostcontainerd", | ||
| "name": "hostcontainerd", | ||
| "readOnly": False, | ||
| }, | ||
| { | ||
| "mountPath": "/cert-copy-and-toml-update.sh", | ||
| "name": "cert-copy-and-toml-update", | ||
| "subPath": "update-containerd-certs.sh", | ||
| }, | ||
| { | ||
| "name": "private-ca-cert-foo", | ||
| "mountPath": "/private-ca-certs/private-ca-cert-foo/private-ca-cert-foo.crt", | ||
| "subPath": "private-ca-cert-foo.crt", | ||
| }, | ||
| { | ||
| "name": "private-ca-cert-bar", | ||
| "mountPath": "/private-ca-certs/private-ca-cert-bar/private-ca-cert-bar.crt", | ||
| "subPath": "private-ca-cert-bar.crt", | ||
| }, | ||
| ] | ||
|
|
||
| assert volmounts == volmounts_expected |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters