Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/openbao-migrations.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,5 +43,11 @@ jobs:
steps:
- uses: actions/checkout@v4

- name: Test dependency version comparisons
run: migrations/openbao/tests/verify-openbao-version-test.sh

- name: Test kubectl source-build contract
run: migrations/openbao/tests/verify-kubectl-build-test.sh

- name: Run kv write retry test
run: migrations/openbao/tests/kv-write-retry-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,13 @@ spec:
- name: init-script
configMap:
name: {{ $serverFullname }}-init-script
defaultMode: 0500
# The migrations image runs as the upstream non-root OpenBao user.
# These ConfigMaps contain executable scripts, not secrets.
defaultMode: 0555
- name: utils-script
configMap:
name: {{ $serverFullname }}-utils-script
defaultMode: 0500
defaultMode: 0555
---
apiVersion: v1
kind: ConfigMap
Expand Down
109 changes: 84 additions & 25 deletions migrations/openbao/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,5 +1,32 @@
ARG ALPINE_VERSION=3.23
ARG GO_IMAGE=golang:1.27.0-alpine3.23@sha256:3747dcba41c8b0db3211fda4db61638b980e17ac5bb3c94460a975a9cfe19395
ARG KUBECTL_GO_IMAGE=golang:1.26.6-alpine3.23@sha256:e57c41c1d5864341031181b0db34b9a537bb5773eb6428e4e5bdaea0f9135406
ARG BAO_VERSION=2.6.2
ARG BAO_RUNTIME_SHA256=sha256:11fd73a2102cda9c55d5d881a8c3210303146a7ec1e8ac76f526e175c6d24641
ARG BAO_SOURCE_COMMIT=dd9c19c37a878cf4a81b18efb8d6f0599c7da923
ARG BAO_SOURCE_SHA256=a7784550a9db16f24e99d65a18c9b12a433707c79ef4c1f34262d3f48171c7a9
ARG KUBECTL_VERSION=v1.36.4
ARG KUBECTL_GO_VERSION=go1.26.6
ARG KUBECTL_SOURCE_COMMIT=bb826b1d48562f110659e64e8ec444327433db95
ARG KUBECTL_SOURCE_SHA256=3c28f11492472df48e658551bf268fd92938b127b0f9dcef7090ac800318c821
ARG KUBECTL_BUILD_DATE=2026-08-20T03:09:25Z
ARG KUBECTL_SOURCE_DATE_EPOCH=1787195365

FROM --platform=$BUILDPLATFORM ${GO_IMAGE} AS bao-builder

ARG BAO_VERSION
ARG BAO_SOURCE_COMMIT
ARG BAO_SOURCE_SHA256
ARG TARGETARCH

RUN apk add --no-cache curl xz
COPY scripts/build-openbao.sh scripts/verify-openbao.sh /usr/local/bin/
RUN BAO_VERSION="${BAO_VERSION}" \
BAO_SOURCE_COMMIT="${BAO_SOURCE_COMMIT}" \
BAO_SOURCE_SHA256="${BAO_SOURCE_SHA256}" \
TARGETARCH="${TARGETARCH}" \
OUTPUT_DIR=/out \
/usr/local/bin/build-openbao.sh

FROM --platform=$BUILDPLATFORM ${GO_IMAGE} AS jwker-builder

Expand All @@ -23,40 +50,72 @@ RUN apk add --no-cache curl tar && \
-ldflags="-s -w -buildid= -X main.version=${JWKER_BUILD_VERSION} -X main.commit=${JWKER_COMMIT} -X main.date=${JWKER_BUILD_DATE}" \
-o /out/jwker ./cmd/jwker

FROM --platform=$BUILDPLATFORM alpine:${ALPINE_VERSION} AS kubectl-downloader
FROM --platform=$BUILDPLATFORM ${KUBECTL_GO_IMAGE} AS kubectl-builder

ARG TARGETARCH
ARG KUBECTL_VERSION=v1.36.4
ARG KUBECTL_LINUX_AMD64_SHA256=8b8f088da2dab964f853b38464033b1be15ede2839eca751482357c45abdd05a
ARG KUBECTL_LINUX_ARM64_SHA256=0ecf44450ee6063bf19dd166a103ee6df4a9034455c2abce626e6eea657d73fb

# Verify the official checksum and the reviewed per-architecture checksum.
RUN apk add --no-cache curl && \
case "${TARGETARCH}" in \
amd64) KUBECTL_SHA256="${KUBECTL_LINUX_AMD64_SHA256}" ;; \
arm64) KUBECTL_SHA256="${KUBECTL_LINUX_ARM64_SHA256}" ;; \
*) echo "Unsupported architecture for kubectl: ${TARGETARCH}" >&2; exit 1 ;; \
esac && \
KUBECTL_BASE_URL="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/${TARGETARCH}" && \
mkdir -p /out && \
curl -fsSLo /out/kubectl "${KUBECTL_BASE_URL}/kubectl" && \
curl -fsSLo /tmp/kubectl.sha256 "${KUBECTL_BASE_URL}/kubectl.sha256" && \
test "$(cat /tmp/kubectl.sha256)" = "${KUBECTL_SHA256}" && \
printf '%s %s\n' "${KUBECTL_SHA256}" /out/kubectl | sha256sum -c - && \
chmod +x /out/kubectl

FROM openbao/openbao:2.5.5

# Install runtime dependencies and copy verified tools from downloader stages.
RUN apk add --no-cache openssl uuidgen helm curl bash jq
ARG KUBECTL_VERSION
ARG KUBECTL_GO_VERSION
ARG KUBECTL_SOURCE_COMMIT
ARG KUBECTL_SOURCE_SHA256
ARG KUBECTL_BUILD_DATE
ARG KUBECTL_SOURCE_DATE_EPOCH

# Rebuild the 1.36 client with the patched toolchain while preserving coverage
# across the repository's latest-and-N-2 Kubernetes support window.
RUN apk add --no-cache curl
COPY scripts/build-kubectl.sh scripts/verify-kubectl.sh /usr/local/bin/
RUN KUBECTL_VERSION="${KUBECTL_VERSION}" \
KUBECTL_GO_VERSION="${KUBECTL_GO_VERSION}" \
KUBECTL_SOURCE_COMMIT="${KUBECTL_SOURCE_COMMIT}" \
KUBECTL_SOURCE_SHA256="${KUBECTL_SOURCE_SHA256}" \
KUBECTL_BUILD_DATE="${KUBECTL_BUILD_DATE}" \
KUBECTL_SOURCE_DATE_EPOCH="${KUBECTL_SOURCE_DATE_EPOCH}" \
TARGETARCH="${TARGETARCH}" \
OUTPUT_DIR=/out \
/usr/local/bin/build-kubectl.sh && \
KUBECTL_GO_VERSION="${KUBECTL_GO_VERSION}" \
TARGETARCH="${TARGETARCH}" \
/usr/local/bin/verify-kubectl.sh

FROM alpine:${ALPINE_VERSION} AS kubectl-runtime-verifier

ARG TARGETARCH
ARG KUBECTL_VERSION
ARG KUBECTL_GO_VERSION
ARG KUBECTL_SOURCE_COMMIT
ARG KUBECTL_BUILD_DATE

COPY --from=kubectl-builder /out/kubectl /out/kubectl

# Execute each target binary and verify the public client identity before it is
# copied into the final image.
RUN KUBECTL_METADATA="$(/out/kubectl version --client -o json)" && \
printf '%s\n' "${KUBECTL_METADATA}" | grep -Fq "\"gitVersion\": \"${KUBECTL_VERSION}\"" && \
printf '%s\n' "${KUBECTL_METADATA}" | grep -Fq "\"gitCommit\": \"${KUBECTL_SOURCE_COMMIT}\"" && \
printf '%s\n' "${KUBECTL_METADATA}" | grep -Fq '"gitTreeState": "clean"' && \
printf '%s\n' "${KUBECTL_METADATA}" | grep -Fq "\"buildDate\": \"${KUBECTL_BUILD_DATE}\"" && \
printf '%s\n' "${KUBECTL_METADATA}" | grep -Fq "\"goVersion\": \"${KUBECTL_GO_VERSION}\"" && \
printf '%s\n' "${KUBECTL_METADATA}" | grep -Fq "\"platform\": \"linux/${TARGETARCH}\""

FROM openbao/openbao:${BAO_VERSION}@${BAO_RUNTIME_SHA256}
ARG TARGETARCH

# Upgrade inherited packages before installing runtime dependencies so the
# image receives fixes published after the upstream OpenBao release.
USER root
RUN apk upgrade --no-cache && \
apk add --no-cache openssl uuidgen helm curl bash jq

COPY --from=bao-builder --chmod=0555 /out/bao-linux-${TARGETARCH} /usr/bin/bao
COPY --from=jwker-builder /out/jwker /usr/local/bin/jwker
COPY --from=kubectl-downloader /out/kubectl /usr/local/bin/kubectl
COPY --from=kubectl-runtime-verifier /out/kubectl /usr/local/bin/kubectl

WORKDIR /app

COPY --chmod=775 migrations/ /app/migrations/
COPY --chmod=775 addons/ /app/addons/
COPY --chmod=775 entrypoint.sh /app/

USER openbao

CMD ["/app/entrypoint.sh"]
7 changes: 4 additions & 3 deletions migrations/openbao/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ This repository ships:
- Numbered shell migrations under `migrations/` that run in order against an OpenBao leader
- Helper utilities under `migrations/utils/`
- The `jwker` CLI used by the install pipeline to convert Kubernetes JWKS material to PEM
- Reproducible `jwker` source build and checksum-verified official `kubectl` binary copied from build stages
- Reproducible `jwker` and `kubectl` source builds copied from verified build stages
- Optional addons under `addons/` (e.g., LLS / TURN secret rotation)
- An example Kubernetes Job manifest (`job.yaml`)
- A Docker-based integration test for the helper functions (`tests/`)
Expand Down Expand Up @@ -41,8 +41,9 @@ The shipped `job.yaml` sets a default placeholder value for this variable so the

## Building the container

The `Dockerfile` uses the public upstream OpenBao image (`openbao/openbao:2.5.5`) as the base. To use a different base, edit the `FROM` line directly.
It builds `jwker` v0.2.2 from checksum-pinned source with Go 1.27.0 and downloads the official Kubernetes v1.36.4 `kubectl` binary for the target architecture. The build verifies both the published Kubernetes checksum and the pinned per-architecture checksum.
The `Dockerfile` uses the public upstream OpenBao 2.6.2 image as its runtime base. It replaces the upstream `bao` binary with a reproducible build from the matching checksum-pinned source commit. The build pins x/crypto v0.56.0, gRPC v1.83.1, and go-archive v0.3.0, then verifies those dependency floors and the target architecture from the embedded Go build metadata.

The image also builds `jwker` v0.2.2 from checksum-pinned source with Go 1.27.0. It rebuilds Kubernetes v1.36.4 `kubectl` from the checksum-pinned official source archive with a digest-pinned Go 1.26.6 toolchain and vendored dependencies. Keeping the 1.36 client preserves `kubectl`'s supported one-minor skew across this repository's Kubernetes latest-and-N-2 support window (1.35 through 1.37). The build verifies the source identity, embedded Go and target metadata, and the executable client's version, commit, build date, and platform.

```bash
docker build -t <your-registry>/<your-org>/openbao-migrations:<version> .
Expand Down
66 changes: 66 additions & 0 deletions migrations/openbao/scripts/build-kubectl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
#!/bin/sh
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

set -eu

kubectl_version=${KUBECTL_VERSION:-v1.36.4}
kubectl_source_commit=${KUBECTL_SOURCE_COMMIT:-bb826b1d48562f110659e64e8ec444327433db95}
kubectl_source_sha256=${KUBECTL_SOURCE_SHA256:-3c28f11492472df48e658551bf268fd92938b127b0f9dcef7090ac800318c821}
kubectl_build_date=${KUBECTL_BUILD_DATE:-2026-08-20T03:09:25Z}
kubectl_source_date_epoch=${KUBECTL_SOURCE_DATE_EPOCH:-1787195365}
target_arch=${TARGETARCH:?TARGETARCH must be set}
output_dir=${OUTPUT_DIR:-/out}
source_url=${KUBECTL_SOURCE_URL:-"https://dl.k8s.io/${kubectl_version}/kubernetes-src.tar.gz"}

case "$target_arch" in
amd64 | arm64) ;;
*)
echo "unsupported kubectl architecture: $target_arch" >&2
exit 1
;;
esac

work_dir=$(mktemp -d "${TMPDIR:-/tmp}/nvcf-kubectl-source.XXXXXX")
cleanup() {
rm -rf "$work_dir"
}
trap cleanup EXIT INT TERM

archive="$work_dir/kubernetes-src.tar.gz"
source_dir="$work_dir/source"
mkdir -p "$source_dir" "$output_dir"

curl --fail --location --silent --show-error "$source_url" --output "$archive"
printf '%s %s\n' "$kubectl_source_sha256" "$archive" | sha256sum -c -
tar -xzf "$archive" -C "$source_dir"

version_ldflags=""
for package in k8s.io/client-go/pkg/version k8s.io/component-base/version; do
version_ldflags="$version_ldflags -X ${package}.gitVersion=${kubectl_version}"
version_ldflags="$version_ldflags -X ${package}.gitCommit=${kubectl_source_commit}"
version_ldflags="$version_ldflags -X ${package}.gitTreeState=clean"
version_ldflags="$version_ldflags -X ${package}.buildDate=${kubectl_build_date}"
version_ldflags="$version_ldflags -X ${package}.gitMajor=1"
version_ldflags="$version_ldflags -X ${package}.gitMinor=36"
done

(
cd "$source_dir"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
CGO_ENABLED=0 \
GOOS=linux \
GOARCH="$target_arch" \
GOTOOLCHAIN=local \
GOPROXY=off \
SOURCE_DATE_EPOCH="$kubectl_source_date_epoch" \
go build \
-mod=vendor \
-trimpath \
-buildvcs=false \
-tags=selinux,notest,grpcnotrace \
-ldflags="-s -w -buildid=${version_ldflags}" \
-o "$output_dir/kubectl" \
./cmd/kubectl
)

chmod 0555 "$output_dir/kubectl"
73 changes: 73 additions & 0 deletions migrations/openbao/scripts/build-openbao.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
#!/bin/sh
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

set -eu

script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
repo_root=$(CDPATH= cd -- "$script_dir/.." && pwd)

bao_version=${BAO_VERSION:-2.6.2}
bao_source_commit=${BAO_SOURCE_COMMIT:-dd9c19c37a878cf4a81b18efb8d6f0599c7da923}
bao_source_sha256=${BAO_SOURCE_SHA256:-a7784550a9db16f24e99d65a18c9b12a433707c79ef4c1f34262d3f48171c7a9}
bao_commit_date=${BAO_COMMIT_DATE:-2026-08-18T15:43:05Z}
x_crypto_version=${X_CRYPTO_VERSION:-v0.56.0}
grpc_version=${GRPC_VERSION:-v1.83.1}
go_archive_version=${GO_ARCHIVE_VERSION:-v0.3.0}
output_dir=${OUTPUT_DIR:-"$repo_root/files/openbao"}
source_url=${BAO_SOURCE_URL:-"https://github.com/openbao/openbao/releases/download/v${bao_version}/openbao-dist-v${bao_version}.tar.xz"}

if [ -n "${WORK_DIR:-}" ]; then
work_dir=$WORK_DIR
work_dir_is_ours=0
else
work_dir=$(mktemp -d "${TMPDIR:-/tmp}/nvcf-openbao-source.XXXXXX")
work_dir_is_ours=1
fi

cleanup() {
if [ -z "${KEEP_WORK_DIR:-}" ] && [ "$work_dir_is_ours" = "1" ]; then
rm -rf "$work_dir"
else
echo "Keeping work dir: $work_dir"
fi
}
trap cleanup EXIT INT TERM

archive="$work_dir/openbao.tar.xz"
source_dir="$work_dir/source"
mkdir -p "$source_dir" "$output_dir"

curl --fail --location --silent --show-error "$source_url" --output "$archive"
printf '%s %s\n' "$bao_source_sha256" "$archive" | sha256sum -c -
tar -xJf "$archive" --strip-components=1 -C "$source_dir"

(
cd "$source_dir"
GOFLAGS=-mod=mod go get \
"golang.org/x/crypto@${x_crypto_version}" \
"google.golang.org/grpc@${grpc_version}" \
"github.com/moby/go-archive@${go_archive_version}"
GOFLAGS=-mod=mod go mod tidy
GOFLAGS=-mod=mod go mod verify

if [ -n "${TARGETARCH:-}" ]; then
arches=$TARGETARCH
else
arches="amd64 arm64"
fi

for arch in $arches; do
CGO_ENABLED=0 GOOS=linux GOARCH="$arch" go build \
-mod=mod \
-buildvcs=false \
-trimpath \
-tags ui \
-ldflags "-s -w -X github.com/openbao/openbao/version.fullVersion=${bao_version} -X github.com/openbao/openbao/version.GitCommit=${bao_source_commit} -X github.com/openbao/openbao/version.CommitDate=${bao_commit_date}" \
-o "$output_dir/bao-linux-${arch}" \
.
chmod 555 "$output_dir/bao-linux-${arch}"
done
)

BAO_DIR="$output_dir" ARCHES="${TARGETARCH:-amd64 arm64}" "$script_dir/verify-openbao.sh"
34 changes: 34 additions & 0 deletions migrations/openbao/scripts/verify-kubectl.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#!/bin/sh
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

set -eu

kubectl=${KUBECTL_BINARY:-/out/kubectl}
kubectl_go_version=${KUBECTL_GO_VERSION:-go1.26.6}
target_arch=${TARGETARCH:?TARGETARCH must be set}
metadata=$(mktemp)
trap 'rm -f "$metadata"' EXIT INT TERM

go version -m "$kubectl" >"$metadata"

actual_go_version=$(awk 'NR == 1 { print $2 }' "$metadata")
path=$(awk '$1 == "path" { print $2 }' "$metadata")
goos=$(awk '$1 == "build" && $2 ~ /^GOOS=/ { sub(/^GOOS=/, "", $2); print $2 }' "$metadata")
goarch=$(awk '$1 == "build" && $2 ~ /^GOARCH=/ { sub(/^GOARCH=/, "", $2); print $2 }' "$metadata")
cgo_enabled=$(awk '$1 == "build" && $2 ~ /^CGO_ENABLED=/ { sub(/^CGO_ENABLED=/, "", $2); print $2 }' "$metadata")

if [ "$actual_go_version" != "$kubectl_go_version" ]; then
echo "$kubectl embeds $actual_go_version; expected $kubectl_go_version" >&2
exit 1
fi
if [ "$path" != "k8s.io/kubernetes/cmd/kubectl" ]; then
echo "$kubectl has unexpected module path: $path" >&2
exit 1
fi
if [ "$goos" != "linux" ] || [ "$goarch" != "$target_arch" ] || [ "$cgo_enabled" != "0" ]; then
echo "$kubectl has unexpected target metadata: GOOS=$goos GOARCH=$goarch CGO_ENABLED=$cgo_enabled" >&2
exit 1
fi

echo "verified $kubectl: $kubectl_go_version linux/$target_arch"
Loading
Loading