Change podman log-driver from journald to k8s-file

* When podman runs the Kubelet container, logging to journald means log lines are duplicated in the journal. journalctl -u kubelet shows Kubelet's logs and the same log messages from podman. Using the k8s-file driver alleviates this problem * Fix Kubelet and etcd-member logs to be more readable and reduce unneccessary Kubelet log volume
poseidon · Aug 28, 2022 · 76d92e9 · 76d92e9
1 parent 275fc0f
commit 76d92e9
Show file tree

Hide file tree

Showing 11 changed files with 23 additions and 0 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -4,6 +4,14 @@ Notable changes between versions.
 
 ## Latest
 
+* Kubernetes [v1.25.0](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.25.md#v1250)
+  * Disable LocalStorageCapacityIsolationFSQuotaMonitoring feature gate ([#1220](https://github.com/poseidon/typhoon/pull/1220))
+
+### Fedora CoreOS
+
+* Change Podman `log-driver` from `journald` to `k8s-file` ([#1221](https://github.com/poseidon/typhoon/pull/1221))
+  * Fix `etcd-member` and Kubelet systemd service log lines appearing twice in journal logs
+
 ## v1.24.4
 
 * Kubernetes [v1.24.4](https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.24.md#v1244)

diff --git a/aws/fedora-coreos/kubernetes/butane/controller.yaml b/aws/fedora-coreos/kubernetes/butane/controller.yaml
@@ -18,6 +18,7 @@ systemd:
         ExecStartPre=-/usr/bin/podman rm etcd
         ExecStart=/usr/bin/podman run --name etcd \
           --env-file /etc/etcd/etcd.env \
+          --log-driver k8s-file \
           --network host \
           --volume /var/lib/etcd:/var/lib/etcd:rw,Z \
           --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
@@ -66,6 +67,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml b/aws/fedora-coreos/kubernetes/workers/butane/worker.yaml
@@ -39,6 +39,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/azure/fedora-coreos/kubernetes/butane/controller.yaml b/azure/fedora-coreos/kubernetes/butane/controller.yaml
@@ -18,6 +18,7 @@ systemd:
         ExecStartPre=-/usr/bin/podman rm etcd
         ExecStart=/usr/bin/podman run --name etcd \
           --env-file /etc/etcd/etcd.env \
+          --log-driver k8s-file \
           --network host \
           --volume /var/lib/etcd:/var/lib/etcd:rw,Z \
           --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
@@ -62,6 +63,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml b/azure/fedora-coreos/kubernetes/workers/butane/worker.yaml
@@ -35,6 +35,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml b/bare-metal/fedora-coreos/kubernetes/butane/controller.yaml
@@ -18,6 +18,7 @@ systemd:
         ExecStartPre=-/usr/bin/podman rm etcd
         ExecStart=/usr/bin/podman run --name etcd \
           --env-file /etc/etcd/etcd.env \
+          --log-driver k8s-file \
           --network host \
           --volume /var/lib/etcd:/var/lib/etcd:rw,Z \
           --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
@@ -61,6 +62,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml b/bare-metal/fedora-coreos/kubernetes/butane/worker.yaml
@@ -34,6 +34,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/controller.yaml
@@ -18,6 +18,7 @@ systemd:
         ExecStartPre=-/usr/bin/podman rm etcd
         ExecStart=/usr/bin/podman run --name etcd \
           --env-file /etc/etcd/etcd.env \
+          --log-driver k8s-file \
           --network host \
           --volume /var/lib/etcd:/var/lib/etcd:rw,Z \
           --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
@@ -64,6 +65,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml b/digital-ocean/fedora-coreos/kubernetes/butane/worker.yaml
@@ -38,6 +38,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml b/google-cloud/fedora-coreos/kubernetes/butane/controller.yaml
@@ -18,6 +18,7 @@ systemd:
         ExecStartPre=-/usr/bin/podman rm etcd
         ExecStart=/usr/bin/podman run --name etcd \
           --env-file /etc/etcd/etcd.env \
+          --log-driver k8s-file \
           --network host \
           --volume /var/lib/etcd:/var/lib/etcd:rw,Z \
           --volume /etc/ssl/etcd:/etc/ssl/certs:ro,Z \
@@ -62,6 +63,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \

diff --git a/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml b/google-cloud/fedora-coreos/kubernetes/workers/butane/worker.yaml
@@ -35,6 +35,7 @@ systemd:
         ExecStartPre=/usr/bin/bash -c "grep 'certificate-authority-data' /etc/kubernetes/kubeconfig | awk '{print $2}' | base64 -d > /etc/kubernetes/ca.crt"
         ExecStartPre=-/usr/bin/podman rm kubelet
         ExecStart=/usr/bin/podman run --name kubelet \
+          --log-driver k8s-file \
           --privileged \
           --pid host \
           --network host \