From cf7aecce4a1f30e09885a04aa86df9738bf99978 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 1 May 2026 01:41:50 -0700 Subject: [PATCH 1/2] feat(package): add Debian packaging support --- .github/workflows/deb-package.yml | 92 +++++++ .github/workflows/driver-vm-linux.yml | 254 +++++++++++++++++++ .github/workflows/release-dev.yml | 47 +++- .github/workflows/release-tag.yml | 32 ++- .gitignore | 3 + .markdownlint-cli2.jsonc | 7 +- architecture/gateway.md | 7 +- crates/openshell-driver-vm/README.md | 6 +- crates/openshell-driver-vm/start.sh | 2 +- crates/openshell-server/src/cli.rs | 43 +++- crates/openshell-server/src/compute/vm.rs | 10 +- crates/openshell-server/src/lib.rs | 7 - deploy/deb/control.in | 16 ++ deploy/deb/openshell-gateway.service | 28 +++ deploy/deb/postinst.sh | 34 +++ deploy/deb/postrm.sh | 13 + deploy/deb/prerm.sh | 21 ++ install-dev.sh | 288 ++++++++++++++++++++++ tasks/package.toml | 25 ++ tasks/scripts/package-deb-install.sh | 90 +++++++ tasks/scripts/package-deb.sh | 188 ++++++++++++++ tasks/scripts/release.py | 16 ++ 22 files changed, 1195 insertions(+), 34 deletions(-) create mode 100644 .github/workflows/deb-package.yml create mode 100644 .github/workflows/driver-vm-linux.yml create mode 100644 deploy/deb/control.in create mode 100644 deploy/deb/openshell-gateway.service create mode 100755 deploy/deb/postinst.sh create mode 100755 deploy/deb/postrm.sh create mode 100755 deploy/deb/prerm.sh create mode 100755 install-dev.sh create mode 100644 tasks/package.toml create mode 100755 tasks/scripts/package-deb-install.sh create mode 100755 tasks/scripts/package-deb.sh diff --git a/.github/workflows/deb-package.yml b/.github/workflows/deb-package.yml new file mode 100644 index 000000000..75c206def --- /dev/null +++ b/.github/workflows/deb-package.yml @@ -0,0 +1,92 @@ +name: Debian Package + +on: + workflow_call: + inputs: + deb-version: + required: true + type: string + checkout-ref: + required: true + type: string + +permissions: + contents: read + packages: read + +defaults: + run: + shell: bash + +jobs: + build-deb-linux: + name: Build Debian Package (Linux ${{ matrix.arch }}) + strategy: + matrix: + include: + - arch: amd64 + runner: build-amd64 + deb_arch: amd64 + cli_target: x86_64-unknown-linux-musl + gnu_target: x86_64-unknown-linux-gnu + - arch: arm64 + runner: build-arm64 + deb_arch: arm64 + cli_target: aarch64-unknown-linux-musl + gnu_target: aarch64-unknown-linux-gnu + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v6 + with: + ref: ${{ inputs['checkout-ref'] }} + + - name: Download CLI artifact + uses: actions/download-artifact@v4 + with: + name: cli-linux-${{ matrix.arch }} + path: package-input/ + + - name: Download gateway artifact + uses: actions/download-artifact@v4 + with: + name: gateway-binary-linux-${{ matrix.arch }} + path: package-input/ + + - name: Download VM driver artifact + uses: actions/download-artifact@v4 + with: + name: driver-vm-linux-${{ matrix.arch }} + path: package-input/ + + - name: Extract package inputs + run: | + set -euo pipefail + mkdir -p package-binaries + tar -xzf "package-input/openshell-${{ matrix.cli_target }}.tar.gz" -C package-binaries + tar -xzf "package-input/openshell-gateway-${{ matrix.gnu_target }}.tar.gz" -C package-binaries + tar -xzf "package-input/openshell-driver-vm-${{ matrix.gnu_target }}.tar.gz" -C package-binaries + ls -lah package-binaries + + - name: Build Debian package + run: | + set -euo pipefail + OPENSHELL_CLI_BINARY="${PWD}/package-binaries/openshell" \ + OPENSHELL_GATEWAY_BINARY="${PWD}/package-binaries/openshell-gateway" \ + OPENSHELL_DRIVER_VM_BINARY="${PWD}/package-binaries/openshell-driver-vm" \ + OPENSHELL_DEB_VERSION="${{ inputs['deb-version'] }}" \ + OPENSHELL_DEB_ARCH="${{ matrix.deb_arch }}" \ + OPENSHELL_OUTPUT_DIR=artifacts \ + tasks/scripts/package-deb.sh + + - name: Upload Debian package artifact + uses: actions/upload-artifact@v4 + with: + name: deb-linux-${{ matrix.arch }} + path: artifacts/*.deb + retention-days: 5 diff --git a/.github/workflows/driver-vm-linux.yml b/.github/workflows/driver-vm-linux.yml new file mode 100644 index 000000000..9f188c260 --- /dev/null +++ b/.github/workflows/driver-vm-linux.yml @@ -0,0 +1,254 @@ +name: Driver VM Linux + +on: + workflow_call: + inputs: + cargo-version: + required: true + type: string + image-tag: + required: true + type: string + checkout-ref: + required: true + type: string + +permissions: + contents: read + packages: read + +defaults: + run: + shell: bash + +jobs: + download-kernel-runtime: + name: Download Kernel Runtime + runs-on: build-amd64 + timeout-minutes: 10 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs['checkout-ref'] }} + + - name: Download Linux runtime tarballs + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + mkdir -p runtime-artifacts + + for platform in linux-aarch64 linux-x86_64; do + asset="vm-runtime-${platform}.tar.zst" + echo "Downloading ${asset}..." + asset_url=$(curl -fsSL \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${GH_TOKEN}" \ + "https://api.github.com/repos/${GITHUB_REPOSITORY}/releases/tags/vm-dev" \ + | jq -r --arg asset "$asset" '.assets[] | select(.name == $asset) | .browser_download_url' \ + | head -n1) + if [ -z "$asset_url" ]; then + echo "::error::No ${asset} asset found on vm-dev release" + exit 1 + fi + curl -fL -o "runtime-artifacts/${asset}" "$asset_url" + done + + ls -lah runtime-artifacts/ + + - name: Verify downloads + run: | + set -euo pipefail + for platform in linux-aarch64 linux-x86_64; do + test -f "runtime-artifacts/vm-runtime-${platform}.tar.zst" + done + + - name: Upload runtime artifacts + uses: actions/upload-artifact@v4 + with: + name: driver-vm-kernel-runtime-tarballs + path: runtime-artifacts/vm-runtime-*.tar.zst + retention-days: 1 + + build-rootfs: + name: Build Rootfs (${{ matrix.arch }}) + strategy: + matrix: + include: + - arch: arm64 + runner: build-arm64 + guest_arch: aarch64 + - arch: amd64 + runner: build-amd64 + guest_arch: x86_64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + volumes: + - /var/run/docker.sock:/var/run/docker.sock + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs['checkout-ref'] }} + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Log in to GHCR + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Install tools + run: mise install --locked + + - name: Install zstd + run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + + - name: Build base rootfs tarball + run: | + set -euo pipefail + crates/openshell-vm/scripts/build-rootfs.sh \ + --base \ + --arch ${{ matrix.guest_arch }} \ + target/rootfs-build + + mkdir -p target/vm-runtime-compressed + tar -C target/rootfs-build -cf - . \ + | zstd -19 -T0 -o target/vm-runtime-compressed/rootfs.tar.zst + + - name: Upload rootfs artifact + uses: actions/upload-artifact@v4 + with: + name: driver-vm-rootfs-${{ matrix.arch }} + path: target/vm-runtime-compressed/rootfs.tar.zst + retention-days: 1 + + build-driver-vm-linux: + name: Build Driver VM (Linux ${{ matrix.arch }}) + needs: [download-kernel-runtime, build-rootfs] + strategy: + matrix: + include: + - arch: arm64 + runner: build-arm64 + target: aarch64-unknown-linux-gnu + platform: linux-aarch64 + - arch: amd64 + runner: build-amd64 + target: x86_64-unknown-linux-gnu + platform: linux-x86_64 + runs-on: ${{ matrix.runner }} + timeout-minutes: 30 + container: + image: ghcr.io/nvidia/openshell/ci:latest + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: --privileged + env: + MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SCCACHE_MEMCACHED_ENDPOINT: ${{ vars.SCCACHE_MEMCACHED_ENDPOINT }} + OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs['checkout-ref'] }} + fetch-depth: 0 + + - name: Mark workspace safe for git + run: git config --global --add safe.directory "$GITHUB_WORKSPACE" + + - name: Fetch tags + run: git fetch --tags --force + + - name: Install tools + run: mise install --locked + + - name: Cache Rust target and registry + uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2 + with: + shared-key: driver-vm-linux-${{ matrix.arch }} + cache-directories: .cache/sccache + cache-targets: "true" + + - name: Install zstd + run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/* + + - name: Download kernel runtime tarball + uses: actions/download-artifact@v4 + with: + name: driver-vm-kernel-runtime-tarballs + path: runtime-download/ + + - name: Download rootfs tarball + uses: actions/download-artifact@v4 + with: + name: driver-vm-rootfs-${{ matrix.arch }} + path: rootfs-download/ + + - name: Stage compressed runtime for embedding + run: | + set -euo pipefail + COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" + mkdir -p "$COMPRESSED_DIR" + + EXTRACT_DIR=$(mktemp -d) + zstd -d "runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" --stdout \ + | tar -xf - -C "$EXTRACT_DIR" + + for file in "$EXTRACT_DIR"/*; do + [ -f "$file" ] || continue + name=$(basename "$file") + [ "$name" = "provenance.json" ] && continue + zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file" + done + + cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst" + ls -lah "$COMPRESSED_DIR" + + - name: Scope workspace to driver-vm crates + run: | + set -euo pipefail + sed -i 's|members = \["crates/\*"\]|members = ["crates/openshell-driver-vm", "crates/openshell-core"]|' Cargo.toml + + - name: Patch workspace version + if: ${{ inputs['cargo-version'] != '' }} + run: | + set -euo pipefail + sed -i -E '/^\[workspace\.package\]/,/^\[/{s/^version[[:space:]]*=[[:space:]]*".*"/version = "'"${{ inputs['cargo-version'] }}"'"/}' Cargo.toml + + - name: Build openshell-driver-vm + run: | + set -euo pipefail + OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \ + mise x -- cargo build --release -p openshell-driver-vm + + - name: sccache stats + if: always() + run: mise x -- sccache --show-stats + + - name: Package binary + run: | + set -euo pipefail + mkdir -p artifacts + tar -czf "artifacts/openshell-driver-vm-${{ matrix.target }}.tar.gz" \ + -C target/release openshell-driver-vm + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: driver-vm-linux-${{ matrix.arch }} + path: artifacts/*.tar.gz + retention-days: 5 diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index b088ceb33..5563a67eb 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -29,6 +29,7 @@ jobs: outputs: python_version: ${{ steps.v.outputs.python }} cargo_version: ${{ steps.v.outputs.cargo }} + deb_version: ${{ steps.v.outputs.deb }} steps: - uses: actions/checkout@v6 with: @@ -46,6 +47,7 @@ jobs: set -euo pipefail echo "python=$(uv run python tasks/scripts/release.py get-version --python)" >> "$GITHUB_OUTPUT" echo "cargo=$(uv run python tasks/scripts/release.py get-version --cargo)" >> "$GITHUB_OUTPUT" + echo "deb=$(uv run python tasks/scripts/release.py get-version --deb)" >> "$GITHUB_OUTPUT" build-gateway: needs: [compute-versions] @@ -615,12 +617,31 @@ jobs: path: artifacts/*.tar.gz retention-days: 5 + build-driver-vm-linux: + name: Build Driver VM Linux + needs: [compute-versions] + uses: ./.github/workflows/driver-vm-linux.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: dev + checkout-ref: ${{ github.sha }} + secrets: inherit + + build-deb: + name: Build Debian Packages + needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-driver-vm-linux] + uses: ./.github/workflows/deb-package.yml + with: + deb-version: ${{ needs.compute-versions.outputs.deb_version }} + checkout-ref: ${{ github.sha }} + secrets: inherit + # --------------------------------------------------------------------------- # Create / update the dev GitHub Release with CLI binaries and wheels # --------------------------------------------------------------------------- release-dev: name: Release Dev - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, build-deb] runs-on: build-amd64 timeout-minutes: 10 outputs: @@ -656,6 +677,13 @@ jobs: path: release/ merge-multiple: true + - name: Download Debian package artifacts + uses: actions/download-artifact@v4 + with: + pattern: deb-linux-* + path: release/ + merge-multiple: true + - name: Capture wheel filenames id: wheel_filenames run: | @@ -672,6 +700,7 @@ jobs: openshell-x86_64-unknown-linux-musl.tar.gz \ openshell-aarch64-unknown-linux-musl.tar.gz \ openshell-aarch64-apple-darwin.tar.gz \ + openshell_*.deb \ *.whl > openshell-checksums-sha256.txt cat openshell-checksums-sha256.txt sha256sum \ @@ -684,7 +713,7 @@ jobs: openshell-sandbox-aarch64-unknown-linux-gnu.tar.gz > openshell-sandbox-checksums-sha256.txt cat openshell-sandbox-checksums-sha256.txt - - name: Prune stale wheel assets from dev release + - name: Prune stale wheel and deb assets from dev release uses: actions/github-script@v7 env: WHEEL_VERSION: ${{ needs.compute-versions.outputs.python_version }} @@ -717,19 +746,22 @@ jobs: } // Delete stale wheels - let kept = 0, deleted = 0; + let kept = 0, deleted = 0, debDeleted = 0; for (const asset of assets) { - if (!asset.name.endsWith('.whl')) continue; - if (asset.name.startsWith(currentPrefix)) { + if (asset.name.endsWith('.deb')) { + core.info(`Deleting stale deb package: ${asset.name} (id=${asset.id})`); + await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); + debDeleted++; + } else if (asset.name.endsWith('.whl') && asset.name.startsWith(currentPrefix)) { core.info(`Keeping current wheel: ${asset.name}`); kept++; - } else { + } else if (asset.name.endsWith('.whl')) { core.info(`Deleting stale wheel: ${asset.name} (id=${asset.id})`); await github.rest.repos.deleteReleaseAsset({ owner, repo, asset_id: asset.id }); deleted++; } } - core.info(`Summary: kept=${kept}, deleted=${deleted}`); + core.info(`Summary: kept_wheels=${kept}, deleted_wheels=${deleted}, deleted_debs=${debDeleted}`); - name: Move dev tag run: | @@ -760,6 +792,7 @@ jobs: release/openshell-x86_64-unknown-linux-musl.tar.gz release/openshell-aarch64-unknown-linux-musl.tar.gz release/openshell-aarch64-apple-darwin.tar.gz + release/openshell_*.deb release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-apple-darwin.tar.gz diff --git a/.github/workflows/release-tag.yml b/.github/workflows/release-tag.yml index 453636444..7df792cba 100644 --- a/.github/workflows/release-tag.yml +++ b/.github/workflows/release-tag.yml @@ -40,6 +40,7 @@ jobs: outputs: python_version: ${{ steps.v.outputs.python }} cargo_version: ${{ steps.v.outputs.cargo }} + deb_version: ${{ steps.v.outputs.deb }} # Semver without 'v' prefix (e.g. 0.6.0), used for image tags and release body semver: ${{ steps.v.outputs.semver }} steps: @@ -60,6 +61,7 @@ jobs: set -euo pipefail echo "python=$(uv run python tasks/scripts/release.py get-version --python)" >> "$GITHUB_OUTPUT" echo "cargo=$(uv run python tasks/scripts/release.py get-version --cargo)" >> "$GITHUB_OUTPUT" + echo "deb=$(uv run python tasks/scripts/release.py get-version --deb)" >> "$GITHUB_OUTPUT" echo "semver=${RELEASE_TAG#v}" >> "$GITHUB_OUTPUT" build-gateway: @@ -642,12 +644,31 @@ jobs: path: artifacts/*.tar.gz retention-days: 5 + build-driver-vm-linux: + name: Build Driver VM Linux + needs: [compute-versions] + uses: ./.github/workflows/driver-vm-linux.yml + with: + cargo-version: ${{ needs.compute-versions.outputs.cargo_version }} + image-tag: ${{ needs.compute-versions.outputs.semver }} + checkout-ref: ${{ inputs.tag || github.ref }} + secrets: inherit + + build-deb: + name: Build Debian Packages + needs: [compute-versions, build-cli-linux, build-gateway-binary-linux, build-driver-vm-linux] + uses: ./.github/workflows/deb-package.yml + with: + deb-version: ${{ needs.compute-versions.outputs.deb_version }} + checkout-ref: ${{ inputs.tag || github.ref }} + secrets: inherit + # --------------------------------------------------------------------------- # Create a tagged GitHub Release with CLI binaries and wheels # --------------------------------------------------------------------------- release: name: Release - needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, tag-ghcr-release] + needs: [compute-versions, build-cli-linux, build-cli-macos, build-gateway-binary-linux, build-gateway-binary-macos, build-supervisor-binary-linux, build-python-wheels-linux, build-python-wheel-macos, tag-ghcr-release, build-deb] runs-on: build-amd64 timeout-minutes: 10 outputs: @@ -685,6 +706,13 @@ jobs: path: release/ merge-multiple: true + - name: Download Debian package artifacts + uses: actions/download-artifact@v4 + with: + pattern: deb-linux-* + path: release/ + merge-multiple: true + - name: Capture wheel filenames id: wheel_filenames run: | @@ -701,6 +729,7 @@ jobs: openshell-x86_64-unknown-linux-musl.tar.gz \ openshell-aarch64-unknown-linux-musl.tar.gz \ openshell-aarch64-apple-darwin.tar.gz \ + openshell_*.deb \ *.whl > openshell-checksums-sha256.txt cat openshell-checksums-sha256.txt sha256sum \ @@ -733,6 +762,7 @@ jobs: release/openshell-x86_64-unknown-linux-musl.tar.gz release/openshell-aarch64-unknown-linux-musl.tar.gz release/openshell-aarch64-apple-darwin.tar.gz + release/openshell_*.deb release/openshell-gateway-x86_64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-unknown-linux-gnu.tar.gz release/openshell-gateway-aarch64-apple-darwin.tar.gz diff --git a/.gitignore b/.gitignore index 1921454b2..915c90d9d 100644 --- a/.gitignore +++ b/.gitignore @@ -190,6 +190,9 @@ deploy/docker/.build/ # SBOM generated output (JSON, CSV) — release artifacts, not committed deploy/sbom/output/ +# Debian package build output (default OPENSHELL_OUTPUT_DIR for tasks/scripts/package-deb.sh) +artifacts/ + # Local mise settings mise.local.toml diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index 3f340fbc2..bed46a1a8 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -10,7 +10,12 @@ ".opencode/**", ".github/**", "THIRD-PARTY-NOTICES/**", - "CLAUDE.md" + "CLAUDE.md", + "target/**", + "e2e/rust/target/**", + "scripts/lint-mermaid/node_modules/**", + ".venv/**", + "artifacts/**" ], "config": { "default": true, diff --git a/architecture/gateway.md b/architecture/gateway.md index e83640a43..8e2724bc6 100644 --- a/architecture/gateway.md +++ b/architecture/gateway.md @@ -118,11 +118,12 @@ The gateway boots in `cli::run_cli` (`crates/openshell-server/src/cli.rs`) and p ## Configuration -All configuration is via CLI flags with environment variable fallbacks. The `--db-url` flag is required. The `--ssh-handshake-secret` flag is required for non-Docker drivers; Docker sandboxes do not receive a handshake secret. +All configuration is via CLI flags with environment variable fallbacks. The `--db-url` flag is required. | Flag | Env Var | Default | Description | |------|---------|---------|-------------| -| `--port` | `OPENSHELL_SERVER_PORT` | `8080` | TCP listen port (binds `0.0.0.0`) | +| `--port` | `OPENSHELL_SERVER_PORT` | `8080` | TCP listen port | +| `--bind-address` | `OPENSHELL_BIND_ADDRESS` | `0.0.0.0` | Address for the main gateway listener | | `--log-level` | `OPENSHELL_LOG_LEVEL` | `info` | Tracing log level filter | | `--tls-cert` | `OPENSHELL_TLS_CERT` | None | Path to PEM certificate file | | `--tls-key` | `OPENSHELL_TLS_KEY` | None | Path to PEM private key file | @@ -136,7 +137,7 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d | `--grpc-endpoint` | `OPENSHELL_GRPC_ENDPOINT` | None | gRPC endpoint reachable from within the cluster (for supervisor callbacks) | | `--drivers` | `OPENSHELL_DRIVERS` | `kubernetes` | Compute backend to use. Current options are `kubernetes`, `docker`, and `vm`. | | `--vm-driver-state-dir` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Host directory for VM sandbox rootfs, console logs, and runtime state | -| `--driver-dir` | `OPENSHELL_DRIVER_DIR` | unset | Override directory for `openshell-driver-vm`. When unset, the gateway searches `~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`, then a sibling binary. | +| `--driver-dir` | `OPENSHELL_DRIVER_DIR` | unset | Override directory for `openshell-driver-vm`. When unset, the gateway searches `~/.local/libexec/openshell`, `/usr/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`, then a sibling binary. | | `--vm-krun-log-level` | `OPENSHELL_VM_KRUN_LOG_LEVEL` | `1` | libkrun log level for VM helper processes | | `--vm-driver-vcpus` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | Default vCPU count for VM sandboxes | | `--vm-driver-mem-mib` | `OPENSHELL_VM_DRIVER_MEM_MIB` | `2048` | Default memory allocation for VM sandboxes in MiB | diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index 39be02676..a36f3ea44 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -37,7 +37,7 @@ mise run gateway:vm First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. To keep the Unix socket path under macOS `SUN_LEN`, `mise run gateway:vm` and `start.sh` default the state dir to `/tmp/openshell-vm-driver-dev-$USER-port-$PORT/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`) unless `OPENSHELL_VM_DRIVER_STATE_DIR` is set. The wrapper auto-registers the gateway with the CLI (`gateway destroy` + `gateway add`) so no manual registration step is needed. When running under `sudo`, it uses `sudo -u $SUDO_USER` for the registration so the config is written under the invoking user's home directory. Re-runs are idempotent. -It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell` or `/usr/local/libexec`. +It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: @@ -51,7 +51,6 @@ Override via environment: ```shell OPENSHELL_SERVER_PORT=9090 \ -OPENSHELL_SSH_HANDSHAKE_SECRET=$(openssl rand -hex 32) \ crates/openshell-driver-vm/start.sh ``` @@ -110,13 +109,12 @@ target/debug/openshell-gateway \ --database-url sqlite:/tmp/openshell-vm-driver-dev-$USER-port-8080/openshell.db \ --driver-dir $PWD/target/debug \ --grpc-endpoint http://host.containers.internal:8080 \ - --ssh-handshake-secret dev-vm-driver-secret \ --ssh-gateway-host 127.0.0.1 \ --ssh-gateway-port 8080 \ --vm-driver-state-dir /tmp/openshell-vm-driver-dev-$USER-port-8080 ``` -The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conventional install locations (`~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`), then a sibling of the gateway binary. +The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conventional install locations (`~/.local/libexec/openshell`, `/usr/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`), then a sibling of the gateway binary. ## Flags diff --git a/crates/openshell-driver-vm/start.sh b/crates/openshell-driver-vm/start.sh index d98bb7b91..0eb305c7d 100755 --- a/crates/openshell-driver-vm/start.sh +++ b/crates/openshell-driver-vm/start.sh @@ -126,7 +126,7 @@ export OPENSHELL_DRIVER_DIR="${DRIVER_DIR}" export OPENSHELL_GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${SERVER_PORT}}" export OPENSHELL_SSH_GATEWAY_HOST="${OPENSHELL_SSH_GATEWAY_HOST:-127.0.0.1}" export OPENSHELL_SSH_GATEWAY_PORT="${OPENSHELL_SSH_GATEWAY_PORT:-${SERVER_PORT}}" -export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-dev-vm-driver-secret}" +export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-}" export OPENSHELL_VM_DRIVER_STATE_DIR="${STATE_DIR}" # Resolve the VM runtime directory (contains vmlinux, virtiofsd, etc.) diff --git a/crates/openshell-server/src/cli.rs b/crates/openshell-server/src/cli.rs index 6e64b596e..ae90c8b34 100644 --- a/crates/openshell-server/src/cli.rs +++ b/crates/openshell-server/src/cli.rs @@ -9,7 +9,7 @@ use openshell_core::ComputeDriverKind; use openshell_core::config::{ DEFAULT_SERVER_PORT, DEFAULT_SSH_HANDSHAKE_SKEW_SECS, DEFAULT_SSH_PORT, }; -use std::net::SocketAddr; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; use tracing::info; use tracing_subscriber::EnvFilter; @@ -22,10 +22,18 @@ use crate::{run_server, tracing_bus::TracingLogBus}; #[command(version = openshell_core::VERSION)] #[command(about = "OpenShell gRPC/HTTP server", long_about = None)] struct Args { - /// Port to bind the server to (all interfaces). + /// Port to bind the server to. #[arg(long, default_value_t = DEFAULT_SERVER_PORT, env = "OPENSHELL_SERVER_PORT")] port: u16, + /// Address to bind the server to. + #[arg( + long, + default_value_t = IpAddr::V4(Ipv4Addr::UNSPECIFIED), + env = "OPENSHELL_BIND_ADDRESS" + )] + bind_address: IpAddr, + /// Port for unauthenticated health endpoints (healthz, readyz). /// Set to 0 to disable the dedicated health listener. #[arg(long, default_value_t = 0, env = "OPENSHELL_HEALTH_PORT")] @@ -138,8 +146,9 @@ struct Args { /// Directory searched for compute-driver binaries (e.g. /// `openshell-driver-vm`) when an explicit binary override isn't /// configured. When unset, the gateway searches - /// `$HOME/.local/libexec/openshell`, `/usr/local/libexec/openshell`, - /// `/usr/local/libexec`, then a sibling of the gateway binary. + /// `$HOME/.local/libexec/openshell`, `/usr/libexec/openshell`, + /// `/usr/local/libexec/openshell`, `/usr/local/libexec`, then a sibling + /// of the gateway binary. #[arg(long, env = "OPENSHELL_DRIVER_DIR")] driver_dir: Option, @@ -286,7 +295,7 @@ async fn run_from_args(args: Args) -> Result<()> { EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), ); - let bind = SocketAddr::from(([0, 0, 0, 0], args.port)); + let bind = SocketAddr::from((args.bind_address, args.port)); let tls = if args.disable_tls { None @@ -428,7 +437,9 @@ fn parse_compute_driver(value: &str) -> std::result::Result Result { @@ -186,7 +187,7 @@ pub fn resolve_compute_driver_bin(vm_config: &VmComputeConfig) -> Result>() .join(", "); Err(Error::config(format!( - "vm compute driver binary not found (searched {searched_display}); install it under --driver-dir / OPENSHELL_DRIVER_DIR, a conventional libexec path such as ~/.local/libexec/openshell or /usr/local/libexec{{,/openshell}}, or place it next to the gateway binary" + "vm compute driver binary not found (searched {searched_display}); install it under --driver-dir / OPENSHELL_DRIVER_DIR, a conventional libexec path such as ~/.local/libexec/openshell, /usr/libexec/openshell, or /usr/local/libexec{{,/openshell}}, or place it next to the gateway binary" ))) } @@ -441,12 +442,13 @@ mod tests { } #[test] - fn resolve_driver_search_dirs_include_usr_local_libexec_fallbacks() { + fn resolve_driver_search_dirs_include_libexec_fallbacks() { let dirs = resolve_driver_search_dirs(&VmComputeConfig { driver_dir: None, ..Default::default() }); + assert!(dirs.contains(&PathBuf::from("/usr/libexec/openshell"))); assert!(dirs.contains(&PathBuf::from("/usr/local/libexec/openshell"))); assert!(dirs.contains(&PathBuf::from("/usr/local/libexec"))); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 18243dc5a..2a18e209a 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -154,13 +154,6 @@ pub async fn run_server( if database_url.is_empty() { return Err(Error::config("database_url is required")); } - let driver = configured_compute_driver(&config)?; - if config.ssh_handshake_secret.is_empty() && driver != ComputeDriverKind::Docker { - return Err(Error::config( - "ssh_handshake_secret is required. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET", - )); - } - let store = Arc::new(Store::connect(database_url).await?); let oidc_cache = if let Some(ref oidc) = config.oidc { diff --git a/deploy/deb/control.in b/deploy/deb/control.in new file mode 100644 index 000000000..9f77f8775 --- /dev/null +++ b/deploy/deb/control.in @@ -0,0 +1,16 @@ +Package: openshell +Version: @VERSION@ +Architecture: @ARCH@ +Maintainer: NVIDIA OpenShell Maintainers +Section: utils +Priority: optional +Pre-Depends: init-system-helpers (>= 1.54~) +Homepage: https://github.com/NVIDIA/OpenShell +Description: Safe, sandboxed runtimes for autonomous AI agents + OpenShell provides host-side command-line and gateway components for + launching and managing policy-enforced AI agent sandboxes. The package + installs the openshell CLI, openshell-gateway daemon, and the VM compute + driver helper, plus a per-user systemd unit for running the gateway. + . + The systemd unit is user-scope (under /usr/lib/systemd/user/). To start + it, run `systemctl --user enable --now openshell-gateway`. diff --git a/deploy/deb/openshell-gateway.service b/deploy/deb/openshell-gateway.service new file mode 100644 index 000000000..4705b547d --- /dev/null +++ b/deploy/deb/openshell-gateway.service @@ -0,0 +1,28 @@ +[Unit] +Description=OpenShell Gateway +Documentation=https://github.com/NVIDIA/OpenShell +After=default.target + +[Service] +Type=simple +StateDirectory=openshell/gateway +# %S resolves to $XDG_STATE_HOME for user services. +Environment=OPENSHELL_BIND_ADDRESS=127.0.0.1 +Environment=OPENSHELL_SERVER_PORT=17670 +Environment=OPENSHELL_DISABLE_TLS=true +Environment=OPENSHELL_DISABLE_GATEWAY_AUTH=true +Environment=OPENSHELL_DB_URL=sqlite:%S/openshell/gateway/openshell.db +Environment=OPENSHELL_DRIVERS=vm +Environment=OPENSHELL_GRPC_ENDPOINT=http://127.0.0.1:17670 +Environment=OPENSHELL_SSH_GATEWAY_HOST=127.0.0.1 +Environment=OPENSHELL_SSH_GATEWAY_PORT=17670 +Environment=OPENSHELL_VM_DRIVER_STATE_DIR=%S/openshell/vm-driver +EnvironmentFile=-%h/.config/openshell/gateway.env +ExecStart=/usr/bin/openshell-gateway +Restart=on-failure +RestartSec=5s +PrivateTmp=true +UMask=0077 + +[Install] +WantedBy=default.target diff --git a/deploy/deb/postinst.sh b/deploy/deb/postinst.sh new file mode 100755 index 000000000..d9784214f --- /dev/null +++ b/deploy/deb/postinst.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# postinst for openshell. +# +# The packaged systemd unit is user-scope (installed under +# /usr/lib/systemd/user/) so dpkg cannot enable or start it on the user's +# behalf. Each user opts in by running: +# systemctl --user daemon-reload +# systemctl --user enable --now openshell-gateway +# +# Tell any running per-user systemd manager to re-scan its unit search +# path so already-logged-in users see the new unit without restarting +# their session. +set -e + +case "$1" in +configure | abort-upgrade | abort-deconfigure | abort-remove) + if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then + systemctl daemon-reload >/dev/null 2>&1 || true + # --global only refreshes per-user managers' generators on next + # session; existing managers pick up new unit files without it. + systemctl --global daemon-reload >/dev/null 2>&1 || true + fi + ;; + +*) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/deploy/deb/postrm.sh b/deploy/deb/postrm.sh new file mode 100755 index 000000000..7e1893abe --- /dev/null +++ b/deploy/deb/postrm.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# postrm for openshell. +set -e + +if [ -d /run/systemd/system ] && command -v systemctl >/dev/null 2>&1; then + systemctl daemon-reload >/dev/null 2>&1 || true + systemctl --global daemon-reload >/dev/null 2>&1 || true +fi + +exit 0 diff --git a/deploy/deb/prerm.sh b/deploy/deb/prerm.sh new file mode 100755 index 000000000..e9099eac6 --- /dev/null +++ b/deploy/deb/prerm.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# prerm for openshell. +# +# Per-user systemd units cannot be stopped from a system-scope dpkg hook. +# Users who want to stop their gateway before removing the package should +# run `systemctl --user stop openshell-gateway` themselves. +set -e + +case "$1" in +remove | upgrade | deconfigure | failed-upgrade) ;; + +*) + echo "prerm called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + +exit 0 diff --git a/install-dev.sh b/install-dev.sh new file mode 100755 index 000000000..f1f850126 --- /dev/null +++ b/install-dev.sh @@ -0,0 +1,288 @@ +#!/bin/sh +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Install the OpenShell development build from the rolling GitHub `dev` release. +# +# This script is intended as a convenient installer for development builds. It +# currently supports Debian packages on Linux amd64 and arm64 only. +# +set -e + +APP_NAME="openshell" +REPO="NVIDIA/OpenShell" +GITHUB_URL="https://github.com/${REPO}" +RELEASE_TAG="dev" +CHECKSUMS_NAME="openshell-checksums-sha256.txt" + +info() { + printf '%s: %s\n' "$APP_NAME" "$*" >&2 +} + +error() { + printf '%s: error: %s\n' "$APP_NAME" "$*" >&2 + exit 1 +} + +usage() { + cat </dev/null 2>&1 +} + +require_cmd() { + if ! has_cmd "$1"; then + error "'$1' is required" + fi +} + +download() { + _url="$1" + _output="$2" + curl -fLsS --retry 3 --max-redirs 5 -o "$_output" "$_url" +} + +download_release_asset() { + _filename="$1" + _output="$2" + + if curl -fLs --retry 3 --max-redirs 5 -o "$_output" \ + "${GITHUB_URL}/releases/download/${RELEASE_TAG}/${_filename}"; then + return 0 + fi + + # GitHub normalizes `~` to `.` in release asset names, while the checksum file + # still records the Debian package filename with `~dev` for correct version + # ordering. Download the normalized asset but verify it against the checksum + # entry for the original package filename. + _normalized="$(printf '%s' "$_filename" | tr '~' '.')" + if [ "$_normalized" != "$_filename" ]; then + if download "${GITHUB_URL}/releases/download/${RELEASE_TAG}/${_normalized}" "$_output"; then + info "using GitHub-normalized asset name ${_normalized}" + return 0 + fi + fi + + return 1 +} + +as_root() { + if [ "$(id -u)" -eq 0 ]; then + "$@" + elif has_cmd sudo; then + sudo "$@" + else + error "this installer needs root privileges; rerun as root or install sudo" + fi +} + +target_user() { + if [ "$(id -u)" -eq 0 ] && [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then + echo "$SUDO_USER" + else + id -un + fi +} + +user_home() { + _user="$1" + if has_cmd getent; then + _home="$(getent passwd "$_user" | awk -F: '{ print $6 }')" + if [ -n "$_home" ]; then + echo "$_home" + return 0 + fi + fi + + if [ "$(id -un)" = "$_user" ]; then + echo "${HOME:-}" + return 0 + fi + + echo "/home/${_user}" +} + +as_target_user() { + _bus="unix:path=${TARGET_RUNTIME_DIR}/bus" + if [ "$(id -u)" -eq "$TARGET_UID" ]; then + env HOME="$TARGET_HOME" XDG_RUNTIME_DIR="$TARGET_RUNTIME_DIR" DBUS_SESSION_BUS_ADDRESS="$_bus" "$@" + elif has_cmd sudo; then + sudo -u "$TARGET_USER" env HOME="$TARGET_HOME" XDG_RUNTIME_DIR="$TARGET_RUNTIME_DIR" DBUS_SESSION_BUS_ADDRESS="$_bus" "$@" + elif has_cmd runuser; then + runuser -u "$TARGET_USER" -- env HOME="$TARGET_HOME" XDG_RUNTIME_DIR="$TARGET_RUNTIME_DIR" DBUS_SESSION_BUS_ADDRESS="$_bus" "$@" + else + error "cannot run user service commands as ${TARGET_USER}; install sudo or run as ${TARGET_USER}" + fi +} + +check_platform() { + if [ "$(uname -s)" != "Linux" ]; then + error "unsupported OS: $(uname -s); dev Debian packages require Linux" + fi + + require_cmd dpkg +} + +get_deb_arch() { + _arch="$(dpkg --print-architecture)" + + case "$_arch" in + amd64|arm64) + echo "$_arch" + ;; + *) + error "no dev Debian package is published for architecture: ${_arch}" + ;; + esac +} + +find_deb_asset() { + _checksums="$1" + _arch="$2" + + awk -v arch="$_arch" ' + $2 ~ "^\\*?openshell_.*_" arch "\\.deb$" { + sub("^\\*", "", $2) + print $2 + exit + } + ' "$_checksums" +} + +verify_checksum() { + _archive="$1" + _checksums="$2" + _filename="$3" + + if has_cmd sha256sum; then + _expected="$(awk -v name="$_filename" '($2 == name || $2 == "*" name) { print $1; exit }' "$_checksums")" + [ -n "$_expected" ] || error "no checksum entry found for ${_filename}" + echo "$_expected $_archive" | sha256sum -c --quiet + elif has_cmd shasum; then + _expected="$(awk -v name="$_filename" '($2 == name || $2 == "*" name) { print $1; exit }' "$_checksums")" + [ -n "$_expected" ] || error "no checksum entry found for ${_filename}" + echo "$_expected $_archive" | shasum -a 256 -c --quiet + else + error "neither 'sha256sum' nor 'shasum' found; cannot verify download integrity" + fi +} + +install_deb_package() { + _deb_path="$1" + + if has_cmd apt-get; then + as_root env DEBIAN_FRONTEND=noninteractive apt-get install -y \ + -o Dpkg::Options::=--force-confdef \ + -o Dpkg::Options::=--force-confnew \ + "$_deb_path" + elif has_cmd apt; then + as_root env DEBIAN_FRONTEND=noninteractive apt install -y \ + -o Dpkg::Options::=--force-confdef \ + -o Dpkg::Options::=--force-confnew \ + "$_deb_path" + else + as_root dpkg --force-confdef --force-confnew -i "$_deb_path" + fi +} + +start_user_gateway() { + info "starting openshell-gateway user service as ${TARGET_USER}..." + + if ! as_target_user systemctl --user daemon-reload; then + info "could not reach the user systemd manager for ${TARGET_USER}" + info "start the gateway later with: systemctl --user enable --now openshell-gateway" + info "then register it with: openshell gateway add http://127.0.0.1:17670 --local --name local" + return 0 + fi + + as_target_user systemctl --user enable --now openshell-gateway + as_target_user systemctl --user is-active --quiet openshell-gateway + + info "registering local gateway as ${TARGET_USER}..." + as_target_user openshell gateway add http://127.0.0.1:17670 --local --name local \ + || as_target_user openshell gateway select local +} + +main() { + while [ "$#" -gt 0 ]; do + case "$1" in + --help) + usage + exit 0 + ;; + *) + error "unknown option: $1" + ;; + esac + shift + done + + require_cmd curl + check_platform + + TARGET_USER="$(target_user)" + TARGET_UID="$(id -u "$TARGET_USER" 2>/dev/null || true)" + [ -n "$TARGET_UID" ] || error "cannot resolve uid for ${TARGET_USER}" + TARGET_HOME="$(user_home "$TARGET_USER")" + if [ "$(id -u)" -eq "$TARGET_UID" ] && [ -n "${XDG_RUNTIME_DIR:-}" ]; then + TARGET_RUNTIME_DIR="$XDG_RUNTIME_DIR" + else + TARGET_RUNTIME_DIR="/run/user/${TARGET_UID}" + fi + + _arch="$(get_deb_arch)" + _tmpdir="$(mktemp -d)" + chmod 0755 "$_tmpdir" + trap 'rm -rf "$_tmpdir"' EXIT + + _checksums_url="${GITHUB_URL}/releases/download/${RELEASE_TAG}/${CHECKSUMS_NAME}" + info "downloading ${RELEASE_TAG} release checksums..." + download "$_checksums_url" "${_tmpdir}/${CHECKSUMS_NAME}" || { + error "failed to download ${_checksums_url}" + } + + _deb_file="$(find_deb_asset "${_tmpdir}/${CHECKSUMS_NAME}" "$_arch")" + if [ -z "$_deb_file" ]; then + error "no dev Debian package found for architecture: ${_arch}" + fi + + _deb_url="${GITHUB_URL}/releases/download/${RELEASE_TAG}/${_deb_file}" + _deb_path="${_tmpdir}/${_deb_file}" + + info "selected ${_deb_file}" + + info "downloading ${_deb_file}..." + download_release_asset "$_deb_file" "$_deb_path" || { + error "failed to download ${_deb_url}" + } + chmod 0644 "$_deb_path" + + info "verifying checksum..." + verify_checksum "$_deb_path" "${_tmpdir}/${CHECKSUMS_NAME}" "$_deb_file" + + info "installing ${_deb_file}..." + install_deb_package "$_deb_path" + info "installed ${APP_NAME} development package" + start_user_gateway +} + +main "$@" diff --git a/tasks/package.toml b/tasks/package.toml new file mode 100644 index 000000000..496f93de2 --- /dev/null +++ b/tasks/package.toml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Distribution package tasks + +["package:deb"] +description = "Build a Debian package from supplied OpenShell binaries" +run = "tasks/scripts/package-deb.sh" +hide = true + +["package:deb:amd64"] +description = "Build an amd64 Debian package from supplied OpenShell binaries" +env = { OPENSHELL_DEB_ARCH = "amd64" } +run = "tasks/scripts/package-deb.sh" +hide = true + +["package:deb:arm64"] +description = "Build an arm64 Debian package from supplied OpenShell binaries" +env = { OPENSHELL_DEB_ARCH = "arm64" } +run = "tasks/scripts/package-deb.sh" +hide = true + +["package:deb:install"] +description = "Build OpenShell from source and install the deb locally (requires sudo)" +run = "tasks/scripts/package-deb-install.sh" diff --git a/tasks/scripts/package-deb-install.sh b/tasks/scripts/package-deb-install.sh new file mode 100755 index 000000000..735ce70a9 --- /dev/null +++ b/tasks/scripts/package-deb-install.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Build OpenShell from source and install the resulting Debian package +# locally for testing. Intended for developers iterating on the deb itself +# or on the gateway-as-a-service flow. +# +# Steps: +# 1. cargo build --release the three binaries that go into the deb. +# 2. Run tasks/scripts/package-deb.sh against those binaries. +# 3. sudo dpkg -i the resulting artifact. +# 4. Start the packaged user gateway service and register it locally. +# +# Usage: +# mise run package:deb:install +# +# Optional env: +# OPENSHELL_DEB_VERSION override the package version (default: 0.0.0-local) +# OPENSHELL_DEB_ARCH override the deb architecture (default: host) +# OPENSHELL_OUTPUT_DIR override the artifact directory (default: artifacts) + +set -euo pipefail + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +cd "$repo_root" + +VERSION="${OPENSHELL_DEB_VERSION:-0.0.0-local}" +OUTPUT_DIR="${OPENSHELL_OUTPUT_DIR:-artifacts}" +ARCH="${OPENSHELL_DEB_ARCH:-$(dpkg --print-architecture 2>/dev/null || uname -m)}" +GATEWAY_NAME="local" +GATEWAY_ENDPOINT="http://127.0.0.1:17670" + +remove_existing_gateway_registration() { + local config_home="${XDG_CONFIG_HOME:-${HOME}/.config}" + local openshell_config_dir="${config_home}/openshell" + local gateway_dir="${openshell_config_dir}/gateways/${GATEWAY_NAME}" + local active_gateway_path="${openshell_config_dir}/active_gateway" + + if [ ! -f "${gateway_dir}/metadata.json" ]; then + return + fi + + echo "==> Removing existing ${GATEWAY_NAME} gateway registration" + rm -f \ + "${gateway_dir}/metadata.json" \ + "${gateway_dir}/edge_token" \ + "${gateway_dir}/cf_token" \ + "${gateway_dir}/oidc_token.json" + + if [ -f "$active_gateway_path" ] && [ "$(cat "$active_gateway_path")" = "$GATEWAY_NAME" ]; then + rm -f "$active_gateway_path" + fi +} + +echo "==> Building release binaries" +cargo build --release \ + -p openshell-cli \ + -p openshell-server \ + -p openshell-driver-vm + +echo "==> Building Debian package" +OPENSHELL_CLI_BINARY="${repo_root}/target/release/openshell" \ + OPENSHELL_GATEWAY_BINARY="${repo_root}/target/release/openshell-gateway" \ + OPENSHELL_DRIVER_VM_BINARY="${repo_root}/target/release/openshell-driver-vm" \ + OPENSHELL_DEB_VERSION="$VERSION" \ + OPENSHELL_DEB_ARCH="$ARCH" \ + OPENSHELL_OUTPUT_DIR="$OUTPUT_DIR" \ + "${repo_root}/tasks/scripts/package-deb.sh" + +deb_path="${OUTPUT_DIR}/openshell_${VERSION}_${ARCH}.deb" +case "$deb_path" in +/*) ;; +*) deb_path="${repo_root}/${deb_path}" ;; +esac + +echo "==> Installing ${deb_path}" +sudo dpkg -i "$deb_path" + +openshell --version +openshell-gateway --version + +echo "==> Starting user gateway service" +systemctl --user daemon-reload +systemctl --user enable --now openshell-gateway +systemctl --user is-active --quiet openshell-gateway + +echo "==> Registering local gateway" +remove_existing_gateway_registration +openshell gateway add "$GATEWAY_ENDPOINT" --local --name "$GATEWAY_NAME" diff --git a/tasks/scripts/package-deb.sh b/tasks/scripts/package-deb.sh new file mode 100755 index 000000000..9d7e3d328 --- /dev/null +++ b/tasks/scripts/package-deb.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Build the openshell Debian package by staging pre-built binaries alongside +# the authoring tree under deploy/deb/, then invoking dpkg-deb --build. +# +# All static content (systemd unit, /etc/default file, maintainer scripts, +# default gateway metadata, control template) lives under deploy/deb/. This +# script only renders the version/arch into control.in and copies files into +# the package staging tree. + +set -euo pipefail + +APP_NAME="openshell" + +usage() { + cat <<'EOF' +Build the openshell Debian package. + +Required environment: + OPENSHELL_CLI_BINARY Path to openshell + OPENSHELL_GATEWAY_BINARY Path to openshell-gateway + OPENSHELL_DRIVER_VM_BINARY Path to openshell-driver-vm + OPENSHELL_DEB_VERSION Debian package version + +Optional environment: + OPENSHELL_DEB_ARCH Debian architecture (amd64 or arm64; defaults to host arch) + OPENSHELL_OUTPUT_DIR Output directory (default: artifacts) +EOF +} + +require_env() { + local name="$1" + if [ -z "${!name:-}" ]; then + echo "error: ${name} is required" >&2 + usage >&2 + exit 2 + fi +} + +stage_binary() { + local src="$1" + local dst="$2" + if [ ! -x "$src" ]; then + echo "error: binary is missing or not executable: ${src}" >&2 + exit 1 + fi + mkdir -p "$(dirname "$dst")" + install -m 0755 "$src" "$dst" +} + +infer_deb_arch() { + if command -v dpkg >/dev/null 2>&1; then + dpkg --print-architecture + return + fi + + case "$(uname -m)" in + x86_64 | amd64) echo "amd64" ;; + aarch64 | arm64) echo "arm64" ;; + *) uname -m ;; + esac +} + +# --------------------------------------------------------------------------- +# Inputs +# --------------------------------------------------------------------------- + +require_env OPENSHELL_CLI_BINARY +require_env OPENSHELL_GATEWAY_BINARY +require_env OPENSHELL_DRIVER_VM_BINARY +require_env OPENSHELL_DEB_VERSION + +OPENSHELL_DEB_ARCH="${OPENSHELL_DEB_ARCH:-$(infer_deb_arch)}" + +case "$OPENSHELL_DEB_ARCH" in +amd64 | arm64) ;; +*) + echo "error: OPENSHELL_DEB_ARCH must be amd64 or arm64, got ${OPENSHELL_DEB_ARCH}" >&2 + exit 2 + ;; +esac + +repo_root="$(cd "$(dirname "$0")/../.." && pwd)" +src_dir="${repo_root}/deploy/deb" +output_dir_input="${OPENSHELL_OUTPUT_DIR:-artifacts}" +case "$output_dir_input" in +/*) output_dir="$output_dir_input" ;; +*) output_dir="${repo_root}/${output_dir_input}" ;; +esac +mkdir -p "$output_dir" + +if [ ! -d "$src_dir" ]; then + echo "error: deb source directory not found: ${src_dir}" >&2 + exit 1 +fi + +package_file="${output_dir}/${APP_NAME}_${OPENSHELL_DEB_VERSION}_${OPENSHELL_DEB_ARCH}.deb" +tmpdir="$(mktemp -d)" +trap 'rm -rf "$tmpdir"' EXIT + +pkgroot="${tmpdir}/pkg" +mkdir -p "$pkgroot/DEBIAN" + +# --------------------------------------------------------------------------- +# Stage the package payload +# --------------------------------------------------------------------------- + +# Binaries. +stage_binary "$OPENSHELL_CLI_BINARY" "$pkgroot/usr/bin/openshell" +stage_binary "$OPENSHELL_GATEWAY_BINARY" "$pkgroot/usr/bin/openshell-gateway" +stage_binary "$OPENSHELL_DRIVER_VM_BINARY" "$pkgroot/usr/libexec/openshell/openshell-driver-vm" + +# Per-user systemd unit. Each user enables it via `systemctl --user`. +install -D -m 0644 "$src_dir/openshell-gateway.service" \ + "$pkgroot/usr/lib/systemd/user/openshell-gateway.service" + +# --------------------------------------------------------------------------- +# DEBIAN/ control directory +# --------------------------------------------------------------------------- + +# Render control from template. +sed \ + -e "s|@VERSION@|${OPENSHELL_DEB_VERSION}|g" \ + -e "s|@ARCH@|${OPENSHELL_DEB_ARCH}|g" \ + "$src_dir/control.in" >"$pkgroot/DEBIAN/control" + +# No conffiles: the package owns no /etc files. Per-user configuration +# lives under $XDG_CONFIG_HOME/openshell/. + +# Maintainer scripts. +install -m 0755 "$src_dir/postinst.sh" "$pkgroot/DEBIAN/postinst" +install -m 0755 "$src_dir/prerm.sh" "$pkgroot/DEBIAN/prerm" +install -m 0755 "$src_dir/postrm.sh" "$pkgroot/DEBIAN/postrm" + +# --------------------------------------------------------------------------- +# Documentation +# --------------------------------------------------------------------------- + +doc_dir="$pkgroot/usr/share/doc/openshell" +mkdir -p "$doc_dir" + +if [ -f "${repo_root}/LICENSE" ]; then + install -m 0644 "${repo_root}/LICENSE" "$doc_dir/copyright" +else + cat >"$doc_dir/copyright" <<'EOF' +OpenShell is distributed under the Apache-2.0 license. +EOF + chmod 0644 "$doc_dir/copyright" +fi + +# Real RFC2822 date so lintian doesn't complain about epoch-zero changelogs. +gzip -n -9 -c >"$doc_dir/changelog.gz" < $(date -uR) +EOF + +# --------------------------------------------------------------------------- +# Build +# --------------------------------------------------------------------------- + +dpkg-deb --build --root-owner-group "$pkgroot" "$package_file" +dpkg-deb --info "$package_file" +dpkg-deb --contents "$package_file" + +# --------------------------------------------------------------------------- +# Smoke tests +# --------------------------------------------------------------------------- + +extract_dir="${tmpdir}/extract" +mkdir -p "$extract_dir" +dpkg-deb -x "$package_file" "$extract_dir" +"$extract_dir/usr/bin/openshell" --version +"$extract_dir/usr/bin/openshell-gateway" --version +"$extract_dir/usr/libexec/openshell/openshell-driver-vm" --version + +if command -v systemd-analyze >/dev/null 2>&1; then + # verify --user catches user-scope-specific issues like StateDirectory= + # resolution and the %h/%S specifiers used in this unit. + systemd-analyze --user verify "$extract_dir/usr/lib/systemd/user/openshell-gateway.service" \ + || echo "warning: systemd-analyze verify failed in the build environment" >&2 +fi + +echo "Wrote ${package_file}" diff --git a/tasks/scripts/release.py b/tasks/scripts/release.py index 57e8fbf51..0e1772b8a 100644 --- a/tasks/scripts/release.py +++ b/tasks/scripts/release.py @@ -19,6 +19,7 @@ class Versions: python: str cargo: str docker: str + deb: str git_tag: str git_sha: str @@ -55,6 +56,12 @@ def _compute_versions() -> Versions: # Docker tags can't contain '+'. docker_version = cargo_version.replace("+", "-") + # Debian versions use '~' so prereleases sort before the eventual release. + deb_version = cargo_version + deb_version = deb_version[1:] if deb_version.startswith("v") else deb_version + deb_version = deb_version.replace("-dev.", "~dev.", 1) + deb_version = f"{deb_version}-1" + git_tag = _git(["describe", "--tags", "--abbrev=0"]) git_sha = _git(["rev-parse", "--short", "HEAD"]) @@ -62,6 +69,7 @@ def _compute_versions() -> Versions: python=python_version, cargo=cargo_version, docker=docker_version, + deb=deb_version, git_tag=git_tag, git_sha=git_sha, ) @@ -75,10 +83,13 @@ def get_version(format: str) -> None: print(versions.cargo) elif format == "docker": print(versions.docker) + elif format == "deb": + print(versions.deb) else: print(f"VERSION_PY={versions.python}") print(f"VERSION_CARGO={versions.cargo}") print(f"VERSION_DOCKER={versions.docker}") + print(f"VERSION_DEB={versions.deb}") print(f"GIT_TAG={versions.git_tag}") print(f"GIT_SHA={versions.git_sha}") @@ -97,6 +108,9 @@ def build_parser() -> argparse.ArgumentParser: get_version_parser.add_argument( "--docker", action="store_true", help="Print Docker version only." ) + get_version_parser.add_argument( + "--deb", action="store_true", help="Print Debian package version only." + ) return parser @@ -112,6 +126,8 @@ def main() -> None: get_version("cargo") elif args.docker: get_version("docker") + elif args.deb: + get_version("deb") else: get_version("all") From 4baeec02fd332a36d4469378dcc28c056bf07ccc Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 1 May 2026 01:52:42 -0700 Subject: [PATCH 2/2] chore(markdown): restore main lint config --- .markdownlint-cli2.jsonc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc index bed46a1a8..3f340fbc2 100644 --- a/.markdownlint-cli2.jsonc +++ b/.markdownlint-cli2.jsonc @@ -10,12 +10,7 @@ ".opencode/**", ".github/**", "THIRD-PARTY-NOTICES/**", - "CLAUDE.md", - "target/**", - "e2e/rust/target/**", - "scripts/lint-mermaid/node_modules/**", - ".venv/**", - "artifacts/**" + "CLAUDE.md" ], "config": { "default": true,