run BMO deployment as non-root

BMO ironic has no reason to run as root. Make it run as "ironic" user. dnsmasq requires elevated capabiities. k8s is missing the feature of ambient capabilities, so it requires us to setcap the binaries with expected capabilities and container must be running with "allowPrivilegeEscalation: true" in the manifest to allow elevation. Read the ambient capabilities KEP for more details: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md Add securityContext to BMO deployment manifest and keepalived component, with correct UIDs and GIDs. This is important to be able to share files via /shared. Modify keepalived image to run as ironic user, which we use the same UID and GID as the ironic-image. This commit requires ironic-image with PR metal3-io/ironic-image#410 to be merged to work.
metal3-io · Mar 17, 2023 · 4908f57 · 4908f57 · CJCShadowsan · Mar 23, 2023
1 parent bc6ca4b
commit 4908f57
Show file tree

Hide file tree

Showing 5 changed files with 236 additions and 142 deletions.
diff --git a/ironic-deployment/base/ironic.yaml b/ironic-deployment/base/ironic.yaml
@@ -19,134 +19,190 @@ spec:
     spec:
       hostNetwork: true
       containers:
-        - name: ironic-dnsmasq
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          securityContext:
-             capabilities:
-               add: ["NET_ADMIN", "NET_RAW"]
-          command:
-            - /bin/rundnsmasq
-          livenessProbe:
-           exec:
-             command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          readinessProbe:
-           exec:
-             command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-        - name: ironic
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          command:
-            - /bin/runironic
-          livenessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          readinessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-        - name: ironic-log-watch
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          command:
-            - /bin/runlogwatch.sh
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-        - name: ironic-inspector
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          readinessProbe:
-            exec:
-              command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            timeoutSeconds: 10
-            successThreshold: 1
-            failureThreshold: 10
-          livenessProbe:
-            exec:
-              command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
-            initialDelaySeconds: 30
-            periodSeconds: 30
-            timeoutSeconds: 10
-            successThreshold: 1
-            failureThreshold: 10
-          command:
-            - /bin/runironic-inspector
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-        - name: ironic-httpd
-          image: quay.io/metal3-io/ironic
-          imagePullPolicy: Always
-          command:
-            - /bin/runhttpd
-          livenessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          readinessProbe:
-           exec:
-             command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
-           initialDelaySeconds: 30
-           periodSeconds: 30
-           timeoutSeconds: 10
-           successThreshold: 1
-           failureThreshold: 10
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
+      - name: ironic-dnsmasq
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        securityContext:
+          # Must be false so dnsmasq may get the capabilities via file caps
+          # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md
+          # allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_ADMIN
+            - NET_BIND_SERVICE
+            - NET_RAW
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
+        command:
+        - /bin/rundnsmasq
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "ss -lun | grep :67 && ss -lun | grep :69"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+      - name: ironic
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        command:
+        - /bin/runironic
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:6385 || curl -sSfk https://127.0.0.1:6385"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
+      - name: ironic-log-watch
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        command:
+        - /bin/runlogwatch.sh
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
+      - name: ironic-inspector
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSf http://127.0.0.1:5050 || curl -sSf -k https://127.0.0.1:5050"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        command:
+        - /bin/runironic-inspector
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 996 # ironic-inspector
+          runAsGroup: 993 # ironicinspector
+      - name: ironic-httpd
+        image: quay.io/metal3-io/ironic
+        imagePullPolicy: Always
+        command:
+        - /bin/runhttpd
+        livenessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        readinessProbe:
+          exec:
+            command: ["sh", "-c", "curl -sSfk http://127.0.0.1:6180/images"]
+          initialDelaySeconds: 30
+          periodSeconds: 30
+          timeoutSeconds: 10
+          successThreshold: 1
+          failureThreshold: 10
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
       initContainers:
-        - name: ironic-ipa-downloader
-          image: quay.io/metal3-io/ironic-ipa-downloader
-          imagePullPolicy: Always
-          command:
-            - /usr/local/bin/get-resource.sh
-          envFrom:
-            - configMapRef:
-                name: ironic-bmo-configmap
-          volumeMounts:
-            - mountPath: /shared
-              name: ironic-data-volume
+      - name: ironic-ipa-downloader
+        image: quay.io/metal3-io/ironic-ipa-downloader
+        imagePullPolicy: Always
+        command:
+        - /usr/local/bin/get-resource.sh
+        envFrom:
+        - configMapRef:
+            name: ironic-bmo-configmap
+        volumeMounts:
+        - mountPath: /shared
+          name: ironic-data-volume
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          privileged: false
+          runAsUser: 997 # ironic
+          runAsGroup: 994 # ironic
       volumes:
-        - name: ironic-data-volume
-          emptyDir: {}
+      - name: ironic-data-volume
+        emptyDir: {}
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+        fsGroup: 994
diff --git a/ironic-deployment/components/keepalived/keepalived_patch.yaml b/ironic-deployment/components/keepalived/keepalived_patch.yaml
@@ -10,8 +10,19 @@ spec:
       - image: quay.io/metal3-io/keepalived
         name: ironic-endpoint-keepalived
         securityContext:
-              capabilities:
-                add: ["NET_ADMIN", "NET_RAW"]
+          # Must be false so dnsmasq may get the capabilities via file caps
+          # KEP: https://github.com/kubernetes/enhancements/blob/master/keps/sig-security/2763-ambient-capabilities/README.md
+          # allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_ADMIN
+            - NET_BROADCAST
+            - NET_RAW
+          privileged: false
+          runAsUser: 65532
+          runAsGroup: 65532
         envFrom:
-          - configMapRef:
-              name: ironic-bmo-configmap
+        - configMapRef:
+            name: ironic-bmo-configmap
diff --git a/resources/keepalived-docker/Dockerfile b/resources/keepalived-docker/Dockerfile
@@ -4,11 +4,15 @@ ARG BASE_IMAGE=ubuntu:22.04
 FROM $BASE_IMAGE
 ARG DEBIAN_FRONTEND=noninteractive
 
-COPY sample.keepalived.conf /etc/keepalived/keepalived.conf
-COPY manage-keepalived.sh manage-keepalived.sh
-
 RUN apt-get -y update && \
     apt-get -y install keepalived && \
     apt-get -y clean
 
-ENTRYPOINT ["/bin/bash", "manage-keepalived.sh"]
+COPY sample.keepalived.conf /etc/keepalived/keepalived.conf
+COPY manage-keepalived.sh manage-keepalived.sh
+COPY configure-nonroot.sh /
+
+RUN /configure-nonroot.sh && \
+    rm /configure-nonroot.sh
+
+CMD ["/bin/bash", "manage-keepalived.sh"]
diff --git a/resources/keepalived-docker/configure-nonroot.sh b/resources/keepalived-docker/configure-nonroot.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eux
+
+# create nonroot image matching the keepalived manifest
+NONROOT_USER="nonroot"
+NONROOT_GROUP="nonroot"
+NONROOT_UID=65532
+NONROOT_GID=65532
+
+# run as non-root, allow editing the keepalive.conf during startup
+groupadd -g "${NONROOT_GID}" "${NONROOT_GROUP}"
+useradd -u "${NONROOT_UID}" -g "${NONROOT_GID}" -m "${NONROOT_USER}"
+
+mkdir -p /run/keepalived
+chown -R root:"${NONROOT_GROUP}" /etc/keepalived /run/keepalived
+chmod 2775 /etc/keepalived /run/keepalived
+chmod 664 /etc/keepalived/keepalived.conf
+
+setcap "cap_net_raw,cap_net_broadcast,cap_net_admin=+eip" /usr/sbin/keepalived