feat: restricted security context

bento-downloader/Dockerfile → container_images/bento-downloader/Dockerfile

@@ -11,3 +11,13 @@ RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud
     && tar -xf google-cloud-cli-410.tar.gz \
     && ./google-cloud-sdk/install.sh \
     && rm google-cloud-cli-410.tar.gz
+
+ARG USERNAME=yetone
+ARG USER_UID=1000
+ARG USER_GID=$USER_UID
+
+# Create the user
+RUN groupadd --gid $USER_GID $USERNAME \
+    && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME
+
+USER $USER_UID:$USER_GID

bento-downloader/Makefile → container_images/bento-downloader/Makefile

@@ -1,4 +1,4 @@
-IMAGE := quay.io/bentoml/bento-downloader:0.0.1
+IMAGE := quay.io/bentoml/bento-downloader:0.0.3
 
 build:
 	docker build -t ${IMAGE} .

container_images/buildah/Dockerfile

@@ -0,0 +1,25 @@
+FROM quay.io/buildah/stable:v1.23.1
+
+RUN touch /etc/subgid /etc/subuid \
+ && chmod g=u /etc/subgid /etc/subuid /etc/passwd \
+ && echo build:10000:65536 > /etc/subuid \
+ && echo build:10000:65536 > /etc/subgid
+
+ENV BUILDAH_ISOLATION=chroot
+
+ARG BENTO_USER=bentoml
+ARG BENTO_USER_UID=1034
+ARG BENTO_USER_GID=1034
+RUN groupadd -g $BENTO_USER_GID -o $BENTO_USER && useradd -m -u $BENTO_USER_UID -g $BENTO_USER_GID -o -r $BENTO_USER
+
+RUN echo $BENTO_USER:$BENTO_USER_UID:$BENTO_USER_GID >> /etc/subuid \
+  && echo $BENTO_USER:$BENTO_USER_UID:$BENTO_USER_GID >> /etc/subgid
+
+RUN echo "export BUILDAH_ISOLATION=chroot" >> /home/build/.bashrc
+
+RUN mkdir -p /home/build/.config/containers \
+ && (echo '[storage]';echo 'driver = "vfs"') > /home/build/.config/containers/storage.conf
+
+USER build
+WORKDIR /home/build
+

container_images/buildah/Makefile

@@ -0,0 +1,5 @@
+IMAGE := quay.io/bentoml/bentoml-buildah:0.0.3
+
+build:
+	docker build -t ${IMAGE} .
+	docker push ${IMAGE}

controllers/resources/bentorequest_controller.go

@@ -653,12 +653,18 @@ const (
 	BentoImageBuildEngineKaniko           BentoImageBuildEngine = "kaniko"
 	BentoImageBuildEngineBuildkit         BentoImageBuildEngine = "buildkit"
 	BentoImageBuildEngineBuildkitRootless BentoImageBuildEngine = "buildkit-rootless"
+	BentoImageBuildEngineBuildah          BentoImageBuildEngine = "buildah"
 )
 
 const (
 	EnvBentoImageBuildEngine = "BENTO_IMAGE_BUILD_ENGINE"
+	EnvRunInOpenshift        = "RUN_IN_OPENSHIFT"
 )
 
+func checkIfRunInOpenshift() bool {
+	return os.Getenv(EnvRunInOpenshift) == commonconsts.KubeLabelValueTrue
+}
+
 func getBentoImageBuildEngine() BentoImageBuildEngine {
 	engine := os.Getenv(EnvBentoImageBuildEngine)
 	if engine == "" {
@@ -1220,8 +1226,10 @@ func (r *BentoRequestReconciler) generateImageBuilderPod(ctx context.Context, op
 	logrus.Infof("Image builder is using the images %v", *internalImages)
 
 	buildEngine := getBentoImageBuildEngine()
+	isRunInOpenshift := checkIfRunInOpenshift()
 
-	privileged := buildEngine != BentoImageBuildEngineBuildkitRootless
+	privileged := buildEngine != BentoImageBuildEngineBuildkitRootless || isRunInOpenshift
+	unprivilegedUID := int64(1034)
 
 	bentoDownloadCommandTemplate, err := template.New("downloadCommand").Parse(`
 set -e
@@ -1242,7 +1250,7 @@ echo "Removing bento tar file..."
 rm /tmp/downloaded.tar
 {{if not .Privileged}}
 echo "Changing directory permission..."
-chown -R 1000:1000 /workspace
+chown -R {{ .UnprivilegedUID }}:{{ .UnprivilegedUID }} /workspace
 {{end}}
 echo "Done"
 	`)
@@ -1260,6 +1268,7 @@ echo "Done"
 		"BentoRepositoryName": bentoRepositoryName,
 		"BentoVersion":        bentoVersion,
 		"Privileged":          privileged,
+		"UnprivilegedUID":     unprivilegedUID,
 	})
 	if err != nil {
 		err = errors.Wrap(err, "failed to execute download command template")
@@ -1301,6 +1310,17 @@ echo "Done"
 		})
 	}
 
+	restrictedSecurityContext := &corev1.SecurityContext{
+		AllowPrivilegeEscalation: pointer.BoolPtr(false),
+		RunAsNonRoot:             pointer.BoolPtr(true),
+		SeccompProfile: &corev1.SeccompProfile{
+			Type: corev1.SeccompProfileTypeRuntimeDefault,
+		},
+		Capabilities: &corev1.Capabilities{
+			Drop: []corev1.Capability{"ALL"},
+		},
+	}
+
 	initContainers := []corev1.Container{
 		{
 			Name:  "bento-downloader",
@@ -1310,9 +1330,10 @@ echo "Done"
 				"-c",
 				bentoDownloadCommand,
 			},
-			VolumeMounts: volumeMounts,
-			Resources:    downloaderContainerResources,
-			EnvFrom:      downloaderContainerEnvFrom,
+			VolumeMounts:    volumeMounts,
+			Resources:       downloaderContainerResources,
+			EnvFrom:         downloaderContainerEnvFrom,
+			SecurityContext: restrictedSecurityContext,
 		},
 	}
 
@@ -1405,7 +1426,7 @@ echo "Removing model tar file..."
 rm /tmp/downloaded.tar
 {{if not .Privileged}}
 echo "Changing directory permission..."
-chown -R 1000:1000 /workspace
+chown -R {{ .UnprivilegedUID }}:{{ .UnprivilegedUID }} /workspace
 {{end}}
 echo "Done"
 `)).Execute(&modelDownloadCommandOutput, map[string]interface{}{
@@ -1416,6 +1437,7 @@ echo "Done"
 			"ModelRepositoryName":    modelRepositoryName,
 			"ModelVersion":           modelVersion,
 			"Privileged":             privileged,
+			"UnprivilegedUID":        unprivilegedUID,
 		})
 		if err != nil {
 			err = errors.Wrap(err, "failed to generate download command")
@@ -1430,9 +1452,10 @@ echo "Done"
 				"-c",
 				modelDownloadCommand,
 			},
-			VolumeMounts: volumeMounts,
-			Resources:    downloaderContainerResources,
-			EnvFrom:      downloaderContainerEnvFrom,
+			VolumeMounts:    volumeMounts,
+			Resources:       downloaderContainerResources,
+			EnvFrom:         downloaderContainerEnvFrom,
+			SecurityContext: restrictedSecurityContext,
 		})
 	}
 
@@ -1568,6 +1591,23 @@ echo "Done"
 		builderImage = internalImages.Buildkit
 	case BentoImageBuildEngineBuildkitRootless:
 		builderImage = internalImages.BuildkitRootless
+	case BentoImageBuildEngineBuildah:
+		builderImage = internalImages.Buildah
+		command = []string{"bash", "-c"}
+		args = []string{
+			fmt.Sprintf(
+				"buildah bud --format=docker --tls-verify=%v -f %s -t %s /workspace/buildcontext && buildah push --tls-verify=%v %s",
+				!dockerRegistryInsecure,
+				dockerFilePath,
+				inClusterImageName,
+				!dockerRegistryInsecure,
+				inClusterImageName,
+			),
+		}
+		envs = append(envs, corev1.EnvVar{
+			Name:  "BUILDAH_ISOLATION",
+			Value: "chroot",
+		})
 	default:
 		err = errors.Errorf("unknown bento image build engine %s", buildEngine)
 		return
@@ -1592,18 +1632,33 @@ echo "Done"
 
 	var builderContainerSecurityContext *corev1.SecurityContext
 
+	//nolint: gocritic
 	if buildEngine == BentoImageBuildEngineBuildkit {
 		builderContainerSecurityContext = &corev1.SecurityContext{
 			Privileged: pointer.BoolPtr(true),
 		}
 	} else if buildEngine == BentoImageBuildEngineBuildkitRootless {
 		kubeAnnotations["container.apparmor.security.beta.kubernetes.io/builder"] = "unconfined"
+		for _, container := range initContainers {
+			kubeAnnotations[fmt.Sprintf("container.apparmor.security.beta.kubernetes.io/%s", container.Name)] = "unconfined"
+		}
 		builderContainerSecurityContext = &corev1.SecurityContext{
 			SeccompProfile: &corev1.SeccompProfile{
 				Type: corev1.SeccompProfileTypeUnconfined,
 			},
-			RunAsUser:  pointer.Int64Ptr(1000),
-			RunAsGroup: pointer.Int64Ptr(1000),
+			RunAsUser:  pointer.Int64Ptr(unprivilegedUID),
+			RunAsGroup: pointer.Int64Ptr(unprivilegedUID),
+		}
+	} else if buildEngine == BentoImageBuildEngineBuildah {
+		kubeAnnotations["openshift.io/scc"] = "anyuid"
+		builderContainerSecurityContext = &corev1.SecurityContext{
+			RunAsUser:  pointer.Int64Ptr(unprivilegedUID),
+			RunAsGroup: pointer.Int64Ptr(unprivilegedUID),
+			Capabilities: &corev1.Capabilities{
+				Drop: []corev1.Capability{
+					"KILL",
+				},
+			},
 		}
 	}
 
@@ -1724,6 +1779,12 @@ echo "Done"
 			Containers: []corev1.Container{
 				container,
 			},
+			SecurityContext: &corev1.PodSecurityContext{
+				RunAsNonRoot: pointer.BoolPtr(true),
+				SeccompProfile: &corev1.SeccompProfile{
+					Type: corev1.SeccompProfileTypeRuntimeDefault,
+				},
+			},
 		},
 	}
 

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/aws/aws-sdk-go v1.44.152
-	github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1
+	github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d
 	github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451
 	github.com/huandu/xstrings v1.3.2
 	github.com/iancoleman/strcase v0.2.0

go.sum

@@ -77,8 +77,8 @@ github.com/aws/aws-sdk-go v1.44.152 h1:L9aaepO8wHB67gwuGD8VgIYH/cmQDxieCt7FeLa0+
 github.com/aws/aws-sdk-go v1.44.152/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1 h1:VgN2DLopHoMaEVOiD8J4bZO0L1BGXLRSmNSs7GkfbHo=
-github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ=
+github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d h1:r+iumKOD+Ri4u2NggQYudjQurIYN2Rs0nOW48faSt9E=
+github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ=
 github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451 h1:FNxCbN61Ev8ea6BXzlfmRUT5CYNmqlOv8zDRGs8ufVE=
 github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451/go.mod h1:q7tt064G8YIiAwQabKyVaKEdSIHYDQA9Oyt+kyCsflU=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

helm/yatai-image-builder/templates/secret-env.yaml

@@ -32,10 +32,13 @@ stringData:
   INTERNAL_IMAGES_KANIKO: {{ .Values.internalImages.kaniko | quote }}
   INTERNAL_IMAGES_BUILDKIT: {{ .Values.internalImages.buildkit | quote }}
   INTERNAL_IMAGES_BUILDKIT_ROOTLESS: {{ .Values.internalImages.buildkitRootless | quote }}
+  INTERNAL_IMAGES_BUILDAH: {{ .Values.internalImages.buildah | quote }}
 
   {{- if .Values.dockerRegistry.useAWSECRWithIAMRole }}
   AWS_ECR_WITH_IAM_ROLE: "true"
   AWS_ECR_REGION: {{ .Values.dockerRegistry.awsECRRegion | quote }}
   {{- end }}
 
   BENTO_IMAGE_BUILD_ENGINE: {{ .Values.bentoImageBuildEngine | quote }}
+
+  RUN_IN_OPENSHIFT: {{ .Values.runInOpenshift | quote }}

helm/yatai-image-builder/values.yaml

@@ -27,16 +27,16 @@ serviceAccount:
 
 podAnnotations: {}
 
-podSecurityContext: {}
-  # fsGroup: 2000
+podSecurityContext:
+  runAsNonRoot: true
+  seccompProfile:
+    type: RuntimeDefault
 
-securityContext: {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
+securityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+    - ALL
 
 service:
   type: ClusterIP
@@ -96,9 +96,11 @@ aws:
   secretAccessKeyExistingSecretKey: ''
 
 internalImages:
-  bentoDownloader: quay.io/bentoml/bento-downloader:0.0.1
+  bentoDownloader: quay.io/bentoml/bento-downloader:0.0.3
   kaniko: quay.io/bentoml/kaniko:1.9.1
   buildkit: quay.io/bentoml/buildkit:master
   buildkitRootless: quay.io/bentoml/buildkit:master-rootless
+  buildah: quay.io/bentoml/bentoml-buildah:0.0.2
 
 bentoImageBuildEngine: kaniko # options: kaniko, buildkit, buildkit-rootless
+runInOpenshift: false

scripts/quick-install-yatai-image-builder.sh

@@ -226,7 +226,8 @@ if [ "${USE_LOCAL_HELM_CHART}" = "true" ]; then
     --set aws.accessKeyID=${AWS_ACCESS_KEY_ID} \
     --set aws.secretAccessKeyExistingSecretName=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME} \
     --set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY} \
-    --set bentoImageBuildEngine=buildkit-rootless
+    --set bentoImageBuildEngine=buildah \
+    --set runInOpenshift=true
 else
   helm_repo_name=bentoml
   helm_repo_url=https://bentoml.github.io/helm-charts

scripts/start-dev.sh

@@ -58,5 +58,9 @@ function trap_handler() {
 trap trap_handler EXIT
 
 echo "⌛ starting yatai-image-builder..."
-env $(kubectl -n yatai-image-builder get secret yatai-image-builder-env -o jsonpath='{.data}' | $jq 'to_entries|map("\(.key)=\(.value|@base64d)")|.[]' | xargs) make run
+env $(kubectl -n yatai-image-builder get secret yatai-image-builder-env -o jsonpath='{.data}' | $jq 'to_entries|map("\(.key)=\(.value|@base64d)")|.[]' | xargs) \
+  BENTO_IMAGE_BUILD_ENGINE=buildah \
+  INTERNAL_IMAGES_BENTO_DOWNLOADER=quay.io/bentoml/bento-downloader:0.0.3 \
+  INTERNAL_IMAGES_BUILDAH=quay.io/bentoml/bentoml-buildah:0.0.2 \
+  make run
 

tests/e2e/e2e_test.go

@@ -49,9 +49,9 @@ var _ = Describe("yatai-image-builder", Ordered, func() {
 		cmd = exec.Command("kubectl", "-n", "yatai-image-builder", "logs", "--tail", "200", "-l", "app.kubernetes.io/name=yatai-image-builder")
 		logs, _ = utils.Run(cmd)
 		fmt.Println(string(logs))
-		By("Cleaning up BentoRequest resources")
-		cmd = exec.Command("kubectl", "delete", "-f", "tests/e2e/example.yaml")
-		_, _ = utils.Run(cmd)
+		// By("Cleaning up BentoRequest resources")
+		// cmd = exec.Command("kubectl", "delete", "-f", "tests/e2e/example.yaml")
+		// _, _ = utils.Run(cmd)
 	})
 
 	Context("BentoRequest Operator", func() {

tests/e2e/installation_test.sh

@@ -2,12 +2,12 @@
 
 set -xe
 
-kubectl create ns yatai-system
-kubectl create ns yatai-image-builder
+kubectl create ns yatai-system || true
+kubectl create ns yatai-image-builder || true
 kubectl create ns yatai || true
 
 echo "🚀 Creating AWS Secret Access Key..."
-kubectl create secret generic aws-secret-access-key --from-literal=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --namespace yatai-image-builder
+kubectl create secret generic aws-secret-access-key --from-literal=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --namespace yatai-image-builder || true
 echo "🚀 Installing yatai-image-builder..."
 YATAI_ENDPOINT='empty' USE_LOCAL_HELM_CHART=true UPGRADE_CRDS=false AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME=aws-secret-access-key AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY=AWS_SECRET_ACCESS_KEY bash ./scripts/quick-install-yatai-image-builder.sh
 echo "yatai-image-builder helm release values:"