From ec2424a012c398b65fbe1c7b5dda852e2a342f23 Mon Sep 17 00:00:00 2001 From: yetone Date: Fri, 6 Jan 2023 10:10:24 +0000 Subject: [PATCH 1/2] feat: restricted security context --- .../bento-downloader}/Dockerfile | 10 +++ .../bento-downloader}/Makefile | 2 +- container_images/buildah/Dockerfile | 12 +++ container_images/buildah/Makefile | 5 ++ .../resources/bentorequest_controller.go | 79 ++++++++++++++++--- go.mod | 2 +- go.sum | 4 +- .../templates/secret-env.yaml | 3 + helm/yatai-image-builder/values.yaml | 22 +++--- scripts/quick-install-yatai-image-builder.sh | 3 +- tests/e2e/e2e_test.go | 6 +- 11 files changed, 119 insertions(+), 29 deletions(-) rename {bento-downloader => container_images/bento-downloader}/Dockerfile (72%) rename {bento-downloader => container_images/bento-downloader}/Makefile (54%) create mode 100644 container_images/buildah/Dockerfile create mode 100644 container_images/buildah/Makefile diff --git a/bento-downloader/Dockerfile b/container_images/bento-downloader/Dockerfile similarity index 72% rename from bento-downloader/Dockerfile rename to container_images/bento-downloader/Dockerfile index 868d137..5b58609 100644 --- a/bento-downloader/Dockerfile +++ b/container_images/bento-downloader/Dockerfile @@ -11,3 +11,13 @@ RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud && tar -xf google-cloud-cli-410.tar.gz \ && ./google-cloud-sdk/install.sh \ && rm google-cloud-cli-410.tar.gz + +ARG USERNAME=yetone +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME \ + && useradd --uid $USER_UID --gid $USER_GID -m $USERNAME + +USER $USER_UID:$USER_GID diff --git a/bento-downloader/Makefile b/container_images/bento-downloader/Makefile similarity index 54% rename from bento-downloader/Makefile rename to container_images/bento-downloader/Makefile index 0e6178c..1cdc94e 100644 --- a/bento-downloader/Makefile +++ b/container_images/bento-downloader/Makefile @@ -1,4 +1,4 @@ -IMAGE := quay.io/bentoml/bento-downloader:0.0.1 +IMAGE := quay.io/bentoml/bento-downloader:0.0.3 build: docker build -t ${IMAGE} . diff --git a/container_images/buildah/Dockerfile b/container_images/buildah/Dockerfile new file mode 100644 index 0000000..91f9604 --- /dev/null +++ b/container_images/buildah/Dockerfile @@ -0,0 +1,12 @@ +FROM quay.io/buildah/stable:v1.23.1 + +RUN touch /etc/subgid /etc/subuid \ + && chmod g=u /etc/subgid /etc/subuid /etc/passwd \ + && echo build:10000:65536 > /etc/subuid \ + && echo build:10000:65536 > /etc/subgid + +ENV BUILDAH_ISOLATION=chroot + +USER build +WORKDIR /home/build + diff --git a/container_images/buildah/Makefile b/container_images/buildah/Makefile new file mode 100644 index 0000000..81a1b3f --- /dev/null +++ b/container_images/buildah/Makefile @@ -0,0 +1,5 @@ +IMAGE := quay.io/bentoml/bentoml-buildah:0.0.1 + +build: + docker build -t ${IMAGE} . + docker push ${IMAGE} diff --git a/controllers/resources/bentorequest_controller.go b/controllers/resources/bentorequest_controller.go index 97dfae6..2dced72 100644 --- a/controllers/resources/bentorequest_controller.go +++ b/controllers/resources/bentorequest_controller.go @@ -653,12 +653,18 @@ const ( BentoImageBuildEngineKaniko BentoImageBuildEngine = "kaniko" BentoImageBuildEngineBuildkit BentoImageBuildEngine = "buildkit" BentoImageBuildEngineBuildkitRootless BentoImageBuildEngine = "buildkit-rootless" + BentoImageBuildEngineBuildah BentoImageBuildEngine = "buildah" ) const ( EnvBentoImageBuildEngine = "BENTO_IMAGE_BUILD_ENGINE" + EnvRunInOpenshift = "RUN_IN_OPENSHIFT" ) +func checkIfRunInOpenshift() bool { + return os.Getenv(EnvRunInOpenshift) == commonconsts.KubeLabelValueTrue +} + func getBentoImageBuildEngine() BentoImageBuildEngine { engine := os.Getenv(EnvBentoImageBuildEngine) if engine == "" { @@ -1220,8 +1226,10 @@ func (r *BentoRequestReconciler) generateImageBuilderPod(ctx context.Context, op logrus.Infof("Image builder is using the images %v", *internalImages) buildEngine := getBentoImageBuildEngine() + isRunInOpenshift := checkIfRunInOpenshift() - privileged := buildEngine != BentoImageBuildEngineBuildkitRootless + privileged := buildEngine != BentoImageBuildEngineBuildkitRootless || isRunInOpenshift + unprivilegedUID := int64(1034) bentoDownloadCommandTemplate, err := template.New("downloadCommand").Parse(` set -e @@ -1242,7 +1250,7 @@ echo "Removing bento tar file..." rm /tmp/downloaded.tar {{if not .Privileged}} echo "Changing directory permission..." -chown -R 1000:1000 /workspace +chown -R {{ .UnprivilegedUID }}:{{ .UnprivilegedUID }} /workspace {{end}} echo "Done" `) @@ -1260,6 +1268,7 @@ echo "Done" "BentoRepositoryName": bentoRepositoryName, "BentoVersion": bentoVersion, "Privileged": privileged, + "UnprivilegedUID": unprivilegedUID, }) if err != nil { err = errors.Wrap(err, "failed to execute download command template") @@ -1301,6 +1310,17 @@ echo "Done" }) } + restrictedSecurityContext := &corev1.SecurityContext{ + AllowPrivilegeEscalation: pointer.BoolPtr(false), + RunAsNonRoot: pointer.BoolPtr(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + } + initContainers := []corev1.Container{ { Name: "bento-downloader", @@ -1310,9 +1330,10 @@ echo "Done" "-c", bentoDownloadCommand, }, - VolumeMounts: volumeMounts, - Resources: downloaderContainerResources, - EnvFrom: downloaderContainerEnvFrom, + VolumeMounts: volumeMounts, + Resources: downloaderContainerResources, + EnvFrom: downloaderContainerEnvFrom, + SecurityContext: restrictedSecurityContext, }, } @@ -1405,7 +1426,7 @@ echo "Removing model tar file..." rm /tmp/downloaded.tar {{if not .Privileged}} echo "Changing directory permission..." -chown -R 1000:1000 /workspace +chown -R {{ .UnprivilegedUID }}:{{ .UnprivilegedUID }} /workspace {{end}} echo "Done" `)).Execute(&modelDownloadCommandOutput, map[string]interface{}{ @@ -1416,6 +1437,7 @@ echo "Done" "ModelRepositoryName": modelRepositoryName, "ModelVersion": modelVersion, "Privileged": privileged, + "UnprivilegedUID": unprivilegedUID, }) if err != nil { err = errors.Wrap(err, "failed to generate download command") @@ -1430,9 +1452,10 @@ echo "Done" "-c", modelDownloadCommand, }, - VolumeMounts: volumeMounts, - Resources: downloaderContainerResources, - EnvFrom: downloaderContainerEnvFrom, + VolumeMounts: volumeMounts, + Resources: downloaderContainerResources, + EnvFrom: downloaderContainerEnvFrom, + SecurityContext: restrictedSecurityContext, }) } @@ -1568,6 +1591,19 @@ echo "Done" builderImage = internalImages.Buildkit case BentoImageBuildEngineBuildkitRootless: builderImage = internalImages.BuildkitRootless + case BentoImageBuildEngineBuildah: + builderImage = internalImages.Buildah + command = []string{"bash", "-c"} + args = []string{ + fmt.Sprintf( + "buildah bud --format=docker --tls-verify=%v -f %s -t %s /workspace/buildcontext && buildah push --tls-verify=%v %s", + !dockerRegistryInsecure, + dockerFilePath, + inClusterImageName, + !dockerRegistryInsecure, + inClusterImageName, + ), + } default: err = errors.Errorf("unknown bento image build engine %s", buildEngine) return @@ -1592,18 +1628,33 @@ echo "Done" var builderContainerSecurityContext *corev1.SecurityContext + //nolint: gocritic if buildEngine == BentoImageBuildEngineBuildkit { builderContainerSecurityContext = &corev1.SecurityContext{ Privileged: pointer.BoolPtr(true), } } else if buildEngine == BentoImageBuildEngineBuildkitRootless { kubeAnnotations["container.apparmor.security.beta.kubernetes.io/builder"] = "unconfined" + for _, container := range initContainers { + kubeAnnotations[fmt.Sprintf("container.apparmor.security.beta.kubernetes.io/%s", container.Name)] = "unconfined" + } builderContainerSecurityContext = &corev1.SecurityContext{ SeccompProfile: &corev1.SeccompProfile{ Type: corev1.SeccompProfileTypeUnconfined, }, - RunAsUser: pointer.Int64Ptr(1000), - RunAsGroup: pointer.Int64Ptr(1000), + RunAsUser: pointer.Int64Ptr(unprivilegedUID), + RunAsGroup: pointer.Int64Ptr(unprivilegedUID), + } + } else if buildEngine == BentoImageBuildEngineBuildah { + kubeAnnotations["openshift.io/scc"] = "anyuid" + builderContainerSecurityContext = &corev1.SecurityContext{ + RunAsUser: pointer.Int64Ptr(unprivilegedUID), + RunAsGroup: pointer.Int64Ptr(unprivilegedUID), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{ + "KILL", + }, + }, } } @@ -1724,6 +1775,12 @@ echo "Done" Containers: []corev1.Container{ container, }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: pointer.BoolPtr(true), + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, }, } diff --git a/go.mod b/go.mod index 1192d3d..5cc812a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/aws/aws-sdk-go v1.44.152 - github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1 + github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451 github.com/huandu/xstrings v1.3.2 github.com/iancoleman/strcase v0.2.0 diff --git a/go.sum b/go.sum index 22b5441..e3d0ec4 100644 --- a/go.sum +++ b/go.sum @@ -77,8 +77,8 @@ github.com/aws/aws-sdk-go v1.44.152 h1:L9aaepO8wHB67gwuGD8VgIYH/cmQDxieCt7FeLa0+ github.com/aws/aws-sdk-go v1.44.152/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1 h1:VgN2DLopHoMaEVOiD8J4bZO0L1BGXLRSmNSs7GkfbHo= -github.com/bentoml/yatai-common v0.0.0-20230108151027-0a54d02e79b1/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ= +github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d h1:r+iumKOD+Ri4u2NggQYudjQurIYN2Rs0nOW48faSt9E= +github.com/bentoml/yatai-common v0.0.0-20230109041943-798ca210a16d/go.mod h1:pox0XYk/bVUwKkadn0XwWHEbJmxSEeN3+HwGA4a8uOQ= github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451 h1:FNxCbN61Ev8ea6BXzlfmRUT5CYNmqlOv8zDRGs8ufVE= github.com/bentoml/yatai-schemas v0.0.0-20221123041958-d3ff9b721451/go.mod h1:q7tt064G8YIiAwQabKyVaKEdSIHYDQA9Oyt+kyCsflU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/helm/yatai-image-builder/templates/secret-env.yaml b/helm/yatai-image-builder/templates/secret-env.yaml index 2925ee7..3e338b3 100644 --- a/helm/yatai-image-builder/templates/secret-env.yaml +++ b/helm/yatai-image-builder/templates/secret-env.yaml @@ -32,6 +32,7 @@ stringData: INTERNAL_IMAGES_KANIKO: {{ .Values.internalImages.kaniko | quote }} INTERNAL_IMAGES_BUILDKIT: {{ .Values.internalImages.buildkit | quote }} INTERNAL_IMAGES_BUILDKIT_ROOTLESS: {{ .Values.internalImages.buildkitRootless | quote }} + INTERNAL_IMAGES_BUILDAH: {{ .Values.internalImages.buildah | quote }} {{- if .Values.dockerRegistry.useAWSECRWithIAMRole }} AWS_ECR_WITH_IAM_ROLE: "true" @@ -39,3 +40,5 @@ stringData: {{- end }} BENTO_IMAGE_BUILD_ENGINE: {{ .Values.bentoImageBuildEngine | quote }} + + RUN_IN_OPENSHIFT: {{ .Values.runInOpenshift | quote }} diff --git a/helm/yatai-image-builder/values.yaml b/helm/yatai-image-builder/values.yaml index 3f1fa1e..8cc0970 100644 --- a/helm/yatai-image-builder/values.yaml +++ b/helm/yatai-image-builder/values.yaml @@ -27,16 +27,16 @@ serviceAccount: podAnnotations: {} -podSecurityContext: {} - # fsGroup: 2000 +podSecurityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL service: type: ClusterIP @@ -96,9 +96,11 @@ aws: secretAccessKeyExistingSecretKey: '' internalImages: - bentoDownloader: quay.io/bentoml/bento-downloader:0.0.1 + bentoDownloader: quay.io/bentoml/bento-downloader:0.0.3 kaniko: quay.io/bentoml/kaniko:1.9.1 buildkit: quay.io/bentoml/buildkit:master buildkitRootless: quay.io/bentoml/buildkit:master-rootless + buildah: quay.io/bentoml/bentoml-buildah:0.0.1 bentoImageBuildEngine: kaniko # options: kaniko, buildkit, buildkit-rootless +runInOpenshift: false diff --git a/scripts/quick-install-yatai-image-builder.sh b/scripts/quick-install-yatai-image-builder.sh index 1d8fc8d..fe4bec8 100755 --- a/scripts/quick-install-yatai-image-builder.sh +++ b/scripts/quick-install-yatai-image-builder.sh @@ -226,7 +226,8 @@ if [ "${USE_LOCAL_HELM_CHART}" = "true" ]; then --set aws.accessKeyID=${AWS_ACCESS_KEY_ID} \ --set aws.secretAccessKeyExistingSecretName=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME} \ --set aws.secretAccessKeyExistingSecretKey=${AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY} \ - --set bentoImageBuildEngine=buildkit-rootless + --set bentoImageBuildEngine=buildah \ + --set runInOpenshift=true else helm_repo_name=bentoml helm_repo_url=https://bentoml.github.io/helm-charts diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index b962b99..147e8ec 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -49,9 +49,9 @@ var _ = Describe("yatai-image-builder", Ordered, func() { cmd = exec.Command("kubectl", "-n", "yatai-image-builder", "logs", "--tail", "200", "-l", "app.kubernetes.io/name=yatai-image-builder") logs, _ = utils.Run(cmd) fmt.Println(string(logs)) - By("Cleaning up BentoRequest resources") - cmd = exec.Command("kubectl", "delete", "-f", "tests/e2e/example.yaml") - _, _ = utils.Run(cmd) + // By("Cleaning up BentoRequest resources") + // cmd = exec.Command("kubectl", "delete", "-f", "tests/e2e/example.yaml") + // _, _ = utils.Run(cmd) }) Context("BentoRequest Operator", func() { From ef36b8f2c14ca91a1da68216a3543d203d6d2d4b Mon Sep 17 00:00:00 2001 From: yetone Date: Tue, 10 Jan 2023 04:02:02 +0000 Subject: [PATCH 2/2] f --- container_images/buildah/Dockerfile | 13 +++++++++++++ container_images/buildah/Makefile | 2 +- controllers/resources/bentorequest_controller.go | 4 ++++ helm/yatai-image-builder/values.yaml | 2 +- scripts/start-dev.sh | 6 +++++- tests/e2e/installation_test.sh | 6 +++--- 6 files changed, 27 insertions(+), 6 deletions(-) diff --git a/container_images/buildah/Dockerfile b/container_images/buildah/Dockerfile index 91f9604..843826c 100644 --- a/container_images/buildah/Dockerfile +++ b/container_images/buildah/Dockerfile @@ -7,6 +7,19 @@ RUN touch /etc/subgid /etc/subuid \ ENV BUILDAH_ISOLATION=chroot +ARG BENTO_USER=bentoml +ARG BENTO_USER_UID=1034 +ARG BENTO_USER_GID=1034 +RUN groupadd -g $BENTO_USER_GID -o $BENTO_USER && useradd -m -u $BENTO_USER_UID -g $BENTO_USER_GID -o -r $BENTO_USER + +RUN echo $BENTO_USER:$BENTO_USER_UID:$BENTO_USER_GID >> /etc/subuid \ + && echo $BENTO_USER:$BENTO_USER_UID:$BENTO_USER_GID >> /etc/subgid + +RUN echo "export BUILDAH_ISOLATION=chroot" >> /home/build/.bashrc + +RUN mkdir -p /home/build/.config/containers \ + && (echo '[storage]';echo 'driver = "vfs"') > /home/build/.config/containers/storage.conf + USER build WORKDIR /home/build diff --git a/container_images/buildah/Makefile b/container_images/buildah/Makefile index 81a1b3f..159e5da 100644 --- a/container_images/buildah/Makefile +++ b/container_images/buildah/Makefile @@ -1,4 +1,4 @@ -IMAGE := quay.io/bentoml/bentoml-buildah:0.0.1 +IMAGE := quay.io/bentoml/bentoml-buildah:0.0.3 build: docker build -t ${IMAGE} . diff --git a/controllers/resources/bentorequest_controller.go b/controllers/resources/bentorequest_controller.go index 2dced72..8e339fb 100644 --- a/controllers/resources/bentorequest_controller.go +++ b/controllers/resources/bentorequest_controller.go @@ -1604,6 +1604,10 @@ echo "Done" inClusterImageName, ), } + envs = append(envs, corev1.EnvVar{ + Name: "BUILDAH_ISOLATION", + Value: "chroot", + }) default: err = errors.Errorf("unknown bento image build engine %s", buildEngine) return diff --git a/helm/yatai-image-builder/values.yaml b/helm/yatai-image-builder/values.yaml index 8cc0970..b882f4b 100644 --- a/helm/yatai-image-builder/values.yaml +++ b/helm/yatai-image-builder/values.yaml @@ -100,7 +100,7 @@ internalImages: kaniko: quay.io/bentoml/kaniko:1.9.1 buildkit: quay.io/bentoml/buildkit:master buildkitRootless: quay.io/bentoml/buildkit:master-rootless - buildah: quay.io/bentoml/bentoml-buildah:0.0.1 + buildah: quay.io/bentoml/bentoml-buildah:0.0.2 bentoImageBuildEngine: kaniko # options: kaniko, buildkit, buildkit-rootless runInOpenshift: false diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index a3ad9d9..f89f30e 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -58,5 +58,9 @@ function trap_handler() { trap trap_handler EXIT echo "⌛ starting yatai-image-builder..." -env $(kubectl -n yatai-image-builder get secret yatai-image-builder-env -o jsonpath='{.data}' | $jq 'to_entries|map("\(.key)=\(.value|@base64d)")|.[]' | xargs) make run +env $(kubectl -n yatai-image-builder get secret yatai-image-builder-env -o jsonpath='{.data}' | $jq 'to_entries|map("\(.key)=\(.value|@base64d)")|.[]' | xargs) \ + BENTO_IMAGE_BUILD_ENGINE=buildah \ + INTERNAL_IMAGES_BENTO_DOWNLOADER=quay.io/bentoml/bento-downloader:0.0.3 \ + INTERNAL_IMAGES_BUILDAH=quay.io/bentoml/bentoml-buildah:0.0.2 \ + make run diff --git a/tests/e2e/installation_test.sh b/tests/e2e/installation_test.sh index d18cd27..afdaf31 100755 --- a/tests/e2e/installation_test.sh +++ b/tests/e2e/installation_test.sh @@ -2,12 +2,12 @@ set -xe -kubectl create ns yatai-system -kubectl create ns yatai-image-builder +kubectl create ns yatai-system || true +kubectl create ns yatai-image-builder || true kubectl create ns yatai || true echo "🚀 Creating AWS Secret Access Key..." -kubectl create secret generic aws-secret-access-key --from-literal=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --namespace yatai-image-builder +kubectl create secret generic aws-secret-access-key --from-literal=AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} --namespace yatai-image-builder || true echo "🚀 Installing yatai-image-builder..." YATAI_ENDPOINT='empty' USE_LOCAL_HELM_CHART=true UPGRADE_CRDS=false AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_NAME=aws-secret-access-key AWS_SECRET_ACCESS_KEY_EXISTING_SECRET_KEY=AWS_SECRET_ACCESS_KEY bash ./scripts/quick-install-yatai-image-builder.sh echo "yatai-image-builder helm release values:"