build: consolidate reproducible builder scripts

.github/workflows/docker-build-check.yml

@@ -11,51 +11,99 @@ on:
     branches: [ master, next, dev-* ]
 
 jobs:
-  kms:
+  gateway:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Write GIT_REV
-        run: git rev-parse HEAD > kms/dstack-app/builder/.GIT_REV
-
-      - name: Build KMS Docker image
+      - name: Build Gateway Docker image
         uses: docker/build-push-action@v5
         with:
-          context: kms/dstack-app/builder
+          context: gateway/dstack-app/builder
           push: false
-          platforms: linux/amd64
+          load: true
+          tags: dstack-gateway-check:latest
           provenance: false
+          build-contexts: |
+            build-shared=build/shared
           build-args: |
             DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
             DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
 
-      - name: Build KMS contracts
+      - name: Verify pinned packages
         run: |
-          cd kms/auth-eth
-          npm ci
-          npx hardhat compile
+          build/shared/verify-pinned-packages.sh dstack-gateway-check:latest \
+            gateway/dstack-app/builder/shared/pinned-packages.txt
 
-  gateway:
+      - name: Build gateway-builder target
+        run: |
+          docker buildx build \
+            --load \
+            --target gateway-builder \
+            --tag gateway-builder-check:latest \
+            --provenance=false \
+            --build-context build-shared=build/shared \
+            --build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
+            --build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
+            gateway/dstack-app/builder
+
+      - name: Verify builder pinned packages
+        run: |
+          build/shared/verify-pinned-packages.sh gateway-builder-check:latest \
+            gateway/dstack-app/builder/shared/builder-pinned-packages.txt
+
+  kms:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build Gateway Docker image
+      - name: Build KMS Docker image
         uses: docker/build-push-action@v5
         with:
-          context: gateway/dstack-app/builder
+          context: kms/dstack-app/builder
           push: false
-          platforms: linux/amd64
+          load: true
+          tags: dstack-kms-check:latest
           provenance: false
+          build-contexts: |
+            build-shared=build/shared
           build-args: |
             DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
+            DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
+
+      - name: Verify pinned packages (qemu stage)
+        run: |
+          build/shared/verify-pinned-packages.sh dstack-kms-check:latest \
+            kms/dstack-app/builder/shared/qemu-pinned-packages.txt
+
+      - name: Build kms-builder target
+        run: |
+          docker buildx build \
+            --load \
+            --target kms-builder \
+            --tag kms-builder-check:latest \
+            --provenance=false \
+            --build-context build-shared=build/shared \
+            --build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
+            --build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
+            kms/dstack-app/builder
+
+      - name: Verify builder pinned packages
+        run: |
+          build/shared/verify-pinned-packages.sh kms-builder-check:latest \
+            kms/dstack-app/builder/shared/builder-pinned-packages.txt
+
+      - name: Build KMS contracts
+        run: |
+          cd kms/auth-eth
+          npm ci
+          npx hardhat compile
 
   verifier:
     runs-on: ubuntu-latest
@@ -71,7 +119,52 @@ jobs:
           context: verifier
           file: verifier/builder/Dockerfile
           push: false
-          platforms: linux/amd64
+          load: true
+          tags: dstack-verifier-check:latest
           provenance: false
+          build-contexts: |
+            build-shared=build/shared
           build-args: |
             DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}
+            DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}
+
+      - name: Verify pinned packages (runtime)
+        run: |
+          build/shared/verify-pinned-packages.sh dstack-verifier-check:latest \
+            verifier/builder/shared/pinned-packages.txt
+
+      - name: Build verifier-builder target
+        run: |
+          docker buildx build \
+            --load \
+            --target verifier-builder \
+            --tag verifier-builder-check:latest \
+            --provenance=false \
+            --file verifier/builder/Dockerfile \
+            --build-context build-shared=build/shared \
+            --build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
+            --build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
+            verifier
+
+      - name: Verify builder pinned packages
+        run: |
+          build/shared/verify-pinned-packages.sh verifier-builder-check:latest \
+            verifier/builder/shared/builder-pinned-packages.txt
+
+      - name: Build acpi-builder target
+        run: |
+          docker buildx build \
+            --load \
+            --target acpi-builder \
+            --tag verifier-acpi-check:latest \
+            --provenance=false \
+            --file verifier/builder/Dockerfile \
+            --build-context build-shared=build/shared \
+            --build-arg "DSTACK_REV=${{ github.event.pull_request.head.sha || github.sha }}" \
+            --build-arg "DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}" \
+            verifier
+
+      - name: Verify qemu pinned packages
+        run: |
+          build/shared/verify-pinned-packages.sh verifier-acpi-check:latest \
+            verifier/builder/shared/qemu-pinned-packages.txt

.github/workflows/gateway-release.yml

@@ -54,6 +54,8 @@ jobs:
           tags: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }}
           platforms: linux/amd64
           provenance: false
+          build-contexts: |
+            build-shared=build/shared
           build-args: |
             DSTACK_REV=${{ env.GIT_REV }}
             SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }}

.github/workflows/kms-release.yml

@@ -38,9 +38,6 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Write GIT_REV
-        run: git rev-parse HEAD > kms/dstack-app/builder/.GIT_REV
-
       - name: Get Git commit timestamps
         run: |
           echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV
@@ -57,6 +54,8 @@ jobs:
           tags: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }}
           platforms: linux/amd64
           provenance: false
+          build-contexts: |
+            build-shared=build/shared
           build-args: |
             DSTACK_REV=${{ env.GIT_REV }}
             DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}

.github/workflows/verifier-release.yml

@@ -54,6 +54,8 @@ jobs:
           tags: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }}
           platforms: linux/amd64
           provenance: false
+          build-contexts: |
+            build-shared=build/shared
           build-args: |
             DSTACK_REV=${{ env.GIT_REV }}
             DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git

.gitignore

@@ -1,7 +1,8 @@
 /target
 /certs
 /build-config.sh
-/build
+/build/*
+!/build/shared/
 generated/
 node_modules/
 /.cargo

REUSE.toml

@@ -32,7 +32,7 @@ path = [
     "sdk/simulator/*.json",
     "sdk/go/go.sum",
     "sdk/go/ratls/go.sum",
-    "kms/dstack-app/builder/shared/kms-pinned-packages.txt",
+    "kms/dstack-app/builder/shared/builder-pinned-packages.txt",
     "kms/dstack-app/builder/shared/qemu-pinned-packages.txt",
     "gateway/dstack-app/builder/shared/builder-pinned-packages.txt",
     "gateway/dstack-app/builder/shared/pinned-packages.txt",

build/shared/build-lib.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Shared build library for reproducible Docker image builds.
+#
+# Expected variables (set by the sourcing script):
+#   REPO_ROOT    - absolute path to the git repo root
+#   CONTEXT_DIR  - Docker build context directory
+#   DOCKERFILE   - path to the Dockerfile
+#   GIT_REV      - git revision to build
+#   DSTACK_SRC_URL - git URL for dstack source
+
+set -euo pipefail
+
+BUILDKIT_VERSION="v0.20.2"
+BUILDKIT_BUILDER="buildkit_20"
+BUILD_SHARED_DIR="$REPO_ROOT/build/shared"
+
+ensure_buildkit() {
+    if ! docker buildx inspect "$BUILDKIT_BUILDER" &>/dev/null; then
+        docker buildx create --use --driver-opt "image=moby/buildkit:$BUILDKIT_VERSION" --name "$BUILDKIT_BUILDER"
+    fi
+}
+
+extract_packages() {
+    local image_name=$1
+    local pkg_list_file=${2:-}
+    if [ -z "$pkg_list_file" ]; then
+        return
+    fi
+    docker run --rm --entrypoint bash "$image_name" \
+        -c "dpkg -l | grep '^ii' | awk '{print \$2\"=\"\$3}' | sort" \
+        >"$pkg_list_file"
+}
+
+docker_build() {
+    local image_name=$1
+    local target=${2:-}
+    local pkg_list_file=${3:-}
+
+    local commit_timestamp
+    commit_timestamp=$(git -C "$REPO_ROOT" show -s --format=%ct "$GIT_REV")
+
+    local args=(
+        --builder "$BUILDKIT_BUILDER"
+        --progress=plain
+        --output "type=docker,name=$image_name,rewrite-timestamp=true"
+        --build-context "build-shared=$BUILD_SHARED_DIR"
+        --build-arg "SOURCE_DATE_EPOCH=$commit_timestamp"
+        --build-arg "DSTACK_REV=$GIT_REV"
+        --build-arg "DSTACK_SRC_URL=$DSTACK_SRC_URL"
+    )
+
+    if [ -n "${NO_CACHE:-}" ]; then
+        args+=(--no-cache)
+    fi
+
+    if [ -n "$target" ]; then
+        args+=(--target "$target")
+    fi
+
+    docker buildx build "${args[@]}" \
+        --file "$DOCKERFILE" \
+        "$CONTEXT_DIR"
+
+    extract_packages "$image_name" "$pkg_list_file"
+}
+
+# Verify that pinned-packages files haven't changed (idempotency check).
+check_clean_tree() {
+    local check_path=$1
+    local rel_path
+    rel_path=$(realpath --relative-to="$REPO_ROOT" "$check_path")
+    local git_status
+    git_status=$(git -C "$REPO_ROOT" status --porcelain -- "$rel_path")
+    if [ -n "$git_status" ]; then
+        echo "The working tree has updates in $rel_path. Commit or stash before re-running." >&2
+        exit 1
+    fi
+}

build/shared/pin-packages.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Pin APT packages to exact versions from a frozen Debian snapshot.
+# Usage: pin-packages.sh <pkg-list-file>
+#
+# This script:
+# 1. Points APT at a frozen snapshot.debian.org mirror (reproducible package sources)
+# 2. Reads package=version pairs from the given file and creates APT pin preferences
+#    with priority 1001 to force exact versions
+
+set -e
+
+PKG_LIST=$1
+SNAPSHOT_DATE=${SNAPSHOT_DATE:-20260317T000000Z}
+
+if [ -z "$PKG_LIST" ]; then
+    echo "Usage: $0 <pkg-list-file>" >&2
+    exit 1
+fi
+
+echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian/${SNAPSHOT_DATE} bookworm main" > /etc/apt/sources.list
+echo "deb [check-valid-until=no] http://snapshot.debian.org/archive/debian-security/${SNAPSHOT_DATE} bookworm-security main" >> /etc/apt/sources.list
+echo 'Acquire::Check-Valid-Until "false";' > /etc/apt/apt.conf.d/10no-check-valid-until
+
+mkdir -p /etc/apt/preferences.d
+while IFS= read -r line; do
+    pkg=$(echo "$line" | cut -d= -f1)
+    ver=$(echo "$line" | cut -d= -f2)
+    if [ -n "$pkg" ] && [ -n "$ver" ]; then
+        printf 'Package: %s\nPin: version %s\nPin-Priority: 1001\n\n' "$pkg" "$ver" >> /etc/apt/preferences.d/pinned-packages
+    fi
+done < "$PKG_LIST"

build/shared/verify-pinned-packages.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# SPDX-FileCopyrightText: © 2025 Phala Network <dstack@phala.network>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# Verify that installed packages in a Docker image match the committed
+# pinned-packages file. Detects when Dockerfile changes cause package
+# drift without regenerating the pinned-packages list.
+#
+# Usage: verify-pinned-packages.sh <image> <pinned-packages-file>
+
+set -euo pipefail
+
+IMAGE=$1
+PKG_FILE=$2
+
+if [ -z "$IMAGE" ] || [ -z "$PKG_FILE" ]; then
+    echo "Usage: $0 <image> <pinned-packages-file>" >&2
+    exit 1
+fi
+
+ACTUAL=$(docker run --rm --entrypoint bash "$IMAGE" \
+    -c "dpkg -l | grep '^ii' | awk '{print \$2\"=\"\$3}' | sort")
+
+EXPECTED=$(sort "$PKG_FILE")
+
+if [ "$ACTUAL" = "$EXPECTED" ]; then
+    echo "OK: packages in $IMAGE match $PKG_FILE"
+    exit 0
+fi
+
+echo "ERROR: packages in $IMAGE differ from $PKG_FILE" >&2
+echo "" >&2
+diff --unified <(echo "$EXPECTED") <(echo "$ACTUAL") >&2 || true
+echo "" >&2
+echo "Regenerate pinned packages by running the service's build-image.sh" >&2
+exit 1