feat(runner): arbitrary host uid support via keep-id uid remap

agent-image/devenv.nix

@@ -68,13 +68,15 @@ in
 
       # Identity, matched to the Go runtime rather than devenv's default. The
       # runtime runs the agent as uid 1000 with $HOME=/home/agent
-      # (cmd/compass-runner/main.go:48 `-home-dir`, :118 `UID: defaultAgentUID`;
-      # internal/runner/spec.go:29-31 `SpecDefaults.CheckoutDir/HomeDir/UID`), and
-      # launches containers with plain `--userns=keep-id`
-      # (internal/runtime/podman.go:357), which maps the host Runner uid through
-      # unchanged rather than remapping it — hence the `verifyRunnerUID` guard
-      # that fails fast when the Runner is not itself uid 1000
-      # (cmd/compass-runner/main.go:146-169). devenv defaults to user `user` with
+      # (cmd/compass-runner/main.go `-home-dir`, `UID: defaultAgentUID`;
+      # internal/runner/spec.go `SpecDefaults.CheckoutDir/HomeDir/UID`), and
+      # launches containers with
+      # `--userns=keep-id:uid=<agent-uid>,gid=<agent-gid>`
+      # (internal/runtime/podman.go createArgs), which remaps the invoking host
+      # uid to the baked agent uid rather than passing it through — so an
+      # arbitrary host uid still yields an agent that owns /nix. A startup
+      # podman-version preflight (VerifyUsernsRemapSupport, ≥ 4.3) guards that
+      # the engine supports the remap. devenv defaults to user `user` with
       # $HOME=/env; the uid agrees either way — that is what /nix ownership keys
       # on — but the passwd row and $HOME must match what the Runner execs with,
       # or nix/direnv/devenv hit "$HOME is not owned by you" and silently fall

go/cmd/compass-runner/main.go

@@ -86,11 +86,15 @@ func run() error {
 	slog.SetDefault(slog.New(slog.NewTextHandler(os.Stderr, nil)))
 	log := slog.Default()
 
-	// Ahead of every operator-input check: this validates the process's own
-	// identity, takes no configuration, and its failure is unconditional. Behind
-	// the flag checks, an operator on the wrong uid is told to set a token, fixes
-	// that, re-runs, and only then learns the process can never work as this user.
-	if err := verifyRunnerUID(os.Getuid()); err != nil {
+	// Ahead of every operator-input check: this validates an engine fact the
+	// whole launch path depends on — that podman is new enough for the
+	// container userns remap (--userns=keep-id:uid=,gid=, podman ≥ 4.3;
+	// docs/designs/platform/compass-runner-arbitrary-uid/design.md). It takes no
+	// operator configuration and its failure is unconditional. Behind the flag
+	// checks, an operator on too-old a podman is told to set a token, fixes
+	// that, re-runs, and only then learns the engine can never launch a
+	// container — so the legible startup refusal must come first.
+	if err := runtime.NewPodmanCLI().VerifyUsernsRemapSupport(context.Background()); err != nil {
 		return err
 	}
 
@@ -162,31 +166,6 @@ func run() error {
 // container, matching the runtime package's agent-user convention.
 const defaultAgentUID uint32 = 1000
 
-// verifyRunnerUID enforces the baked-uid invariant the agent image and the
-// container runtime jointly depend on. The image bakes the agent user, /nix and
-// $HOME as uid defaultAgentUID, and the containers are launched with podman's
-// plain --userns=keep-id, which maps the host uid through unchanged rather than
-// remapping it. A Runner running as any other uid therefore produces a container
-// whose agent is that uid and so does not own /nix or its own home — the
-// agent-managed devenv then fails deep inside the first nix build. Fail here
-// instead, where the cause is visible.
-//
-// The caller passes the REAL uid (os.Getuid), deliberately, not the effective
-// one: keep-id maps the invoking process's real uid into the container, so that
-// is the uid the agent ends up as. A setuid-style effective-uid difference must
-// therefore not satisfy this guard.
-func verifyRunnerUID(uid int) error {
-	if uid == int(defaultAgentUID) {
-		return nil
-	}
-	return fmt.Errorf(
-		"the runner must run as uid %d, but it is running as uid %d: the agent "+
-			"image bakes the agent user, /nix and $HOME as uid %d, and podman's "+
-			"--userns=keep-id maps the host uid into the container unchanged, so "+
-			"an agent launched by this runner would not own /nix",
-		defaultAgentUID, uid, defaultAgentUID)
-}
-
 // orEnv returns flagVal when non-empty, else the named environment variable.
 func orEnv(flagVal, envKey string) string {
 	if flagVal != "" {

go/cmd/compass-runner/main_test.go

@@ -9,28 +9,6 @@ import (
 	"github.com/sealedsecurity/compass/go/internal/runtime"
 )
 
-// The Runner's uid is a load-bearing precondition, not a preference: the agent
-// image bakes /nix and $HOME as defaultAgentUID and podman's --userns=keep-id
-// maps the host uid through unchanged, so a Runner on any other uid launches an
-// agent that cannot write /nix. The guard must reject that at startup, and its
-// message must name the invariant — an operator who only sees "permission
-// denied" from a nix build three layers down cannot act on it.
-func TestVerifyRunnerUID(t *testing.T) {
-	if err := verifyRunnerUID(int(defaultAgentUID)); err != nil {
-		t.Fatalf("verifyRunnerUID(%d) = %v, want nil", defaultAgentUID, err)
-	}
-
-	err := verifyRunnerUID(int(defaultAgentUID) + 1)
-	if err == nil {
-		t.Fatalf("verifyRunnerUID(%d) = nil, want an error", defaultAgentUID+1)
-	}
-	for _, want := range []string{"1000", "1001", "/nix", "keep-id"} {
-		if !strings.Contains(err.Error(), want) {
-			t.Errorf("verifyRunnerUID error %q does not name %q", err, want)
-		}
-	}
-}
-
 // parseMount is the operator surface for --mount: a malformed value must be
 // rejected at flag-parse with a message an operator can act on (it names the bad
 // input and the host:container[:ro] shape), and a well-formed value must reach

go/internal/runtime/agent.go

@@ -248,6 +248,7 @@ func (r *AgentRuntime) createAndStart(ctx context.Context, spec AgentSpec) (Cont
 		Name:   spec.Name,
 		CapAdd: []string{capNetAdmin},
 		Mounts: spec.Mounts,
+		UID:    spec.Workspace.UID,
 		// Keep the container alive so the Runner can exec into it; the agent is
 		// driven via exec, not as the container's main process.
 		Command: []string{"sleep", "infinity"},
@@ -279,8 +280,9 @@ func (r *AgentRuntime) provision(ctx context.Context, id ContainerID, spec Agent
 	return r.ensureCheckoutDir(ctx, id, spec.Workspace)
 }
 
-// armEgress arms the egress firewall as root (needs NET_ADMIN). After this, the
-// agent user — with no capabilities — cannot alter the ruleset.
+// armEgress arms the egress firewall as the image's default user (uid 1000)
+// with CAP_NET_ADMIN. After this, an agent exec — run as the agent uid with no
+// capabilities — cannot alter the ruleset.
 func (r *AgentRuntime) armEgress(ctx context.Context, id ContainerID, egress EgressPolicy) error {
 	out, err := r.runtime.Exec(ctx, id, NewExecSpec("sh", "-c", egress.NftScript()))
 	if err != nil {

go/internal/runtime/egress_integrity_podman_test.go

@@ -0,0 +1,111 @@
+//go:build podman
+
+package runtime
+
+// Egress-integrity boundary proof against real rootless podman: an agent exec —
+// run as the agent uid with an explicit --user — holds an EMPTY effective
+// capability set inside a NET_ADMIN container, so it cannot alter the egress
+// ruleset the container's privileged entrypoint armed (egress.go:6-10, "the
+// agent then runs as a non-root user whose capability set is empty, so it cannot
+// flush or edit the ruleset even though the container nominally holds the
+// capability").
+//
+// What this pins: podman's own --user mechanism strips the container's ambient
+// CAP_NET_ADMIN from an exec pinned to the agent uid — the property the hermetic
+// spec tests (agentenv_test.go) cannot prove because they never spawn podman. It
+// does NOT prove any production call site sets --user: those are pinned
+// separately by agentenv_test.go TestExecSpecRunsAsWorkspaceUIDNotContainerRoot
+// (the streaming agent session) and lifecycle_test.go (ExecAsAgent's nft flush
+// is denied). Together the three lock both halves of the boundary: the call
+// sites set --user, and --user actually drops the capability.
+//
+// Skipped (not failed) when podman isn't usable, matching lifecycle_test /
+// config_mount_test. Build-tagged (podman) so it is not part of the hermetic gate.
+
+import (
+	"context"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// capEffOf runs `grep CapEff /proc/self/status` inside container id as the given
+// ExecSpec identity and returns the hex effective-capability mask (the token
+// after "CapEff:"). It drives the real PodmanCLI.Exec so the --user plumbing
+// under test is the one exercised in production.
+func capEffOf(t *testing.T, ctx context.Context, cli *PodmanCLI, id ContainerID, spec ExecSpec) string {
+	t.Helper()
+	out, err := cli.Exec(ctx, id, spec)
+	if err != nil {
+		t.Fatalf("exec CapEff probe: %v", err)
+	}
+	if !out.Success() {
+		t.Fatalf("CapEff probe exited %d: %s", out.ExitCode, out.Stderr)
+	}
+	// /proc/self/status line: "CapEff:\t0000000000000000"
+	for _, line := range strings.Split(out.Stdout, "\n") {
+		if rest, ok := strings.CutPrefix(strings.TrimSpace(line), "CapEff:"); ok {
+			return strings.TrimSpace(rest)
+		}
+	}
+	t.Fatalf("no CapEff line in /proc/self/status: %q", out.Stdout)
+	return ""
+}
+
+// TestAgentExecDropsNetAdminInNetAdminContainer is the egress-integrity
+// regression: in a container granted CAP_NET_ADMIN (as every agent container
+// is, agent.go createAndStart CapAdd), an exec pinned to the agent uid via
+// --user must have an all-zero effective capability set, while the container's
+// default-user exec (the armEgress identity) retains the capability. A regression
+// that drops the --user on an agent exec makes the two masks equal and fails
+// here.
+func TestAgentExecDropsNetAdminInNetAdminContainer(t *testing.T) {
+	if !podmanUsable() {
+		t.Skip("rootless podman not usable in this environment")
+	}
+	ctx := context.Background()
+	cli := NewPodmanCLI()
+
+	// A NET_ADMIN container remapped to the baked agent uid, mirroring a real
+	// agent container's create (agent.go createAndStart).
+	const agentUID uint32 = 1000
+	spec := ContainerSpec{
+		Image:   "docker.io/library/alpine:latest",
+		Name:    "compass-egress-integrity-" + strconv.Itoa(os.Getpid()),
+		UID:     agentUID,
+		CapAdd:  []string{capNetAdmin},
+		Command: []string{"sleep", "infinity"},
+	}
+	// Bring the NET_ADMIN container up, mirroring createStartExec's
+	// create/start/force-rm-teardown (userns_remap_test.go) but holding the id so
+	// we can exec against it twice with different identities.
+	_ = exec.Command("podman", "rm", "--force", spec.Name).Run()
+	t.Cleanup(func() { _ = exec.Command("podman", "rm", "--force", spec.Name).Run() })
+	cid, err := cli.Create(ctx, spec)
+	if err != nil {
+		t.Fatalf("create container: %v", err)
+	}
+	if err := cli.Start(ctx, cid); err != nil {
+		t.Fatalf("start container: %v", err)
+	}
+
+	probe := []string{"grep", "CapEff", "/proc/self/status"}
+
+	// The armEgress identity: nil --user, runs as the image default user and
+	// inherits the container's CAP_NET_ADMIN so it can arm nft.
+	privileged := capEffOf(t, ctx, cli, cid, NewExecSpec(probe...))
+	// The agent-work identity: explicit --user <agent uid>, empty capability set.
+	agent := capEffOf(t, ctx, cli, cid, NewExecSpec(probe...).AsUser(strconv.FormatUint(uint64(agentUID), 10)))
+
+	if agent != "0000000000000000" {
+		t.Fatalf("agent-uid exec CapEff = %q, want an empty set %q: an --user agent exec must hold no capabilities so it cannot alter the egress ruleset (egress.go)", agent, "0000000000000000")
+	}
+	if privileged == agent {
+		t.Fatalf("default-user exec CapEff = %q equals the agent-uid exec's = %q: the NET_ADMIN the entrypoint arms with must not be inherited by agent execs", privileged, agent)
+	}
+	if !strings.Contains(privileged, "1000") {
+		t.Fatalf("default-user exec CapEff = %q, want the CAP_NET_ADMIN bit (0x1000) set — the armEgress identity must retain the capability it arms nft with", privileged)
+	}
+}