Hermes recreation issues

.github/workflows/docker-publish-storefront.yml

@@ -51,7 +51,15 @@ jobs:
           tags: |
             type=semver,pattern={{version}}
             type=semver,pattern={{major}}.{{minor}}
-            type=sha,prefix=
+            # Long SHA: needed by the security-scan step which references
+            # ${{ github.sha }} (40-char). Without it Trivy fails with
+            # MANIFEST_UNKNOWN. Same bug the x402 workflow used to have.
+            type=sha,format=long,prefix=
+            # Short SHA: matches the obol binary's version.GitCommit (set via
+            # ldflags from `git rev-parse --short HEAD`). internal/images.Resolve
+            # uses it to commit-pin the storefront deployment so binary upgrades
+            # actually roll the pod.
+            type=sha,format=short,prefix=
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
           labels: |
             org.opencontainers.image.title=obol-stack-public-storefront

.github/workflows/docker-publish-x402.yml

@@ -91,6 +91,11 @@ jobs:
             # Previously `type=sha,prefix=` produced the 7-char short SHA,
             # causing Trivy to fail with MANIFEST_UNKNOWN on every run.
             type=sha,format=long,prefix=
+            # Also publish the 7-char short SHA as a separate tag. The obol
+            # binary embeds version.GitCommit (short SHA) via ldflags and uses
+            # it through internal/images.Resolve to commit-pin the deployments
+            # it manages. Without this, binary upgrades wouldn't roll the pods.
+            type=sha,format=short,prefix=
             type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/feat/secure-enclave-inference' }}
           labels: |
             org.opencontainers.image.title=${{ matrix.component }}

cmd/obol/agent.go

@@ -227,14 +227,22 @@ func agentWalletCommand(cfg *config.Config) *cli.Command {
 					if err != nil {
 						return err
 					}
-					if target.Runtime != agentruntime.OpenClaw {
-						return errors.New("Hermes wallet backup needs a Hermes-native product decision; use OpenClaw backup only for OpenClaw instances")
+					switch target.Runtime {
+					case agentruntime.Hermes:
+						return hermes.BackupWalletCmd(cfg, target.ID, hermes.BackupWalletOptions{
+							Output:      cmd.String("output"),
+							Passphrase:  cmd.String("passphrase"),
+							HasPassFlag: cmd.IsSet("passphrase"),
+						}, getUI(cmd))
+					case agentruntime.OpenClaw:
+						return openclaw.BackupWalletCmd(cfg, target.ID, openclaw.BackupWalletOptions{
+							Output:      cmd.String("output"),
+							Passphrase:  cmd.String("passphrase"),
+							HasPassFlag: cmd.IsSet("passphrase"),
+						}, getUI(cmd))
+					default:
+						return fmt.Errorf("unsupported runtime %q", target.Runtime)
 					}
-					return openclaw.BackupWalletCmd(cfg, target.ID, openclaw.BackupWalletOptions{
-						Output:      cmd.String("output"),
-						Passphrase:  cmd.String("passphrase"),
-						HasPassFlag: cmd.IsSet("passphrase"),
-					}, getUI(cmd))
 				},
 			},
 			{
@@ -266,15 +274,25 @@ func agentWalletCommand(cfg *config.Config) *cli.Command {
 					if err != nil {
 						return err
 					}
-					if target.Runtime != agentruntime.OpenClaw {
-						return errors.New("Hermes wallet restore needs a Hermes-native product decision; use OpenClaw restore only for OpenClaw instances")
+					switch target.Runtime {
+					case agentruntime.Hermes:
+						return hermes.RestoreWalletCmd(cfg, target.ID, hermes.RestoreWalletOptions{
+							Input:        cmd.String("input"),
+							Passphrase:   cmd.String("passphrase"),
+							HasPassFlag:  cmd.IsSet("passphrase"),
+							Force:        cmd.Bool("force"),
+							ApplyCluster: true,
+						}, getUI(cmd))
+					case agentruntime.OpenClaw:
+						return openclaw.RestoreWalletCmd(cfg, target.ID, openclaw.RestoreWalletOptions{
+							Input:       cmd.String("input"),
+							Passphrase:  cmd.String("passphrase"),
+							HasPassFlag: cmd.IsSet("passphrase"),
+							Force:       cmd.Bool("force"),
+						}, getUI(cmd))
+					default:
+						return fmt.Errorf("unsupported runtime %q", target.Runtime)
 					}
-					return openclaw.RestoreWalletCmd(cfg, target.ID, openclaw.RestoreWalletOptions{
-						Input:       cmd.String("input"),
-						Passphrase:  cmd.String("passphrase"),
-						HasPassFlag: cmd.IsSet("passphrase"),
-						Force:       cmd.Bool("force"),
-					}, getUI(cmd))
 				},
 			},
 		},

cmd/obol/sell.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ObolNetwork/obol-stack/internal/enclave"
 	"github.com/ObolNetwork/obol-stack/internal/erc8004"
 	"github.com/ObolNetwork/obol-stack/internal/hermes"
+	"github.com/ObolNetwork/obol-stack/internal/images"
 	"github.com/ObolNetwork/obol-stack/internal/inference"
 	"github.com/ObolNetwork/obol-stack/internal/kubectl"
 	"github.com/ObolNetwork/obol-stack/internal/monetizeapi"
@@ -1270,7 +1271,7 @@ func buildDemoResources(name string, spec demoSpec, paymentChain string) []map[s
 						"containers": []map[string]any{
 							{
 								"name":            "demo",
-								"image":           "ghcr.io/obolnetwork/demo-server:latest",
+								"image":           images.Resolve("ghcr.io/obolnetwork/demo-server"),
 								"imagePullPolicy": "IfNotPresent",
 								"env":             env,
 								"ports": []map[string]any{

internal/embed/infrastructure/helmfile.yaml

@@ -2,6 +2,20 @@
 # Orchestrates core infrastructure components deployed with every stack
 # Uses Traefik with Gateway API for routing (replaces nginx-ingress)
 
+# Force helm to use SSA with --force-conflicts on every release so that
+# upgrades take ownership of fields written by other field managers
+# (kubectl-client-side-apply from helm's pre-3.13 default, "before-first-apply"
+# synthesised by the apiserver, or runtime writers like obol's auto-config
+# patches). Without this, every `obol stack down`/`up` cycle hits whack-a-mole
+# SSA conflicts on resources helm shares with other writers.
+helmDefaults:
+  args:
+    # `=true` form is required: helm's `--server-side` takes a value
+    # (auto|true|false). Without `=true`, helm consumes the next arg as the
+    # value and rejects `--force-conflicts` as an unknown apply method.
+    - "--server-side=true"
+    - "--force-conflicts"
+
 repositories:
   - name: traefik
     url: https://traefik.github.io/charts

internal/hermes/hermes.go

@@ -33,10 +33,13 @@ const (
 	// renovate: datasource=helm depName=raw registryUrl=https://bedag.github.io/helm-charts/
 	rawChartVersion = "2.0.2"
 
-	defaultImage     = "nousresearch/hermes-agent:v2026.4.23"
-	hermesInstallDir = "/data/.hermes/hermes-agent"
-	hermesRepoURL    = "https://github.com/NousResearch/hermes-agent.git"
-	hermesBinary     = hermesInstallDir + "/venv/bin/hermes"
+	defaultImage = "nousresearch/hermes-agent:v2026.4.23"
+	// hermesBinary points at the venv that the upstream image preinstalls at
+	// build time via `uv pip install -e ".[all]"`. Using the in-image venv
+	// avoids cloning the repo + rebuilding a venv on every cold start, and
+	// keeps the persistent PVC free of the read-only git pack files that
+	// poison subsequent local-path-provisioner chowns on macOS virtiofs.
+	hermesBinary = "/opt/hermes/.venv/bin/hermes"
 
 	containerUID  = 10000
 	containerGID  = 10000
@@ -243,6 +246,9 @@ func Sync(cfg *config.Config, id string, u *ui.UI) error {
 		return fmt.Errorf("helmfile sync failed: %w", err)
 	}
 
+	// Publish wallet-metadata ConfigMap for the frontend (namespace now exists).
+	applyWalletMetadataConfigMap(cfg, id, deploymentDir)
+
 	u.Blank()
 	u.Success("Hermes installed successfully!")
 	u.Detail("Namespace", agentruntime.Namespace(agentruntime.Hermes, id))
@@ -686,6 +692,20 @@ func writeDeploymentFiles(cfg *config.Config, id, deploymentDir, agentBaseURL st
 func generateHelmfile(namespace string) string {
 	return fmt.Sprintf(`# Managed by obol agent
 
+# --server-side --force-conflicts on every helm release so upgrades take
+# ownership of fields previously written by other managers (e.g. helm's
+# pre-3.13 client-side-apply default, the apiserver's synthesised
+# "before-first-apply", or runtime kubectl applies). Without this, every
+# subsequent `+"`obol agent sync`"+` after a fresh install hits whack-a-mole
+# SSA conflicts on the remote-signer Secret labels and similar shared fields.
+helmDefaults:
+  args:
+    # =true form is required: helm's --server-side takes a value
+    # (auto|true|false), so without =true helm consumes the next arg as the
+    # value and rejects --force-conflicts as an unknown apply method.
+    - "--server-side=true"
+    - "--force-conflicts"
+
 repositories:
   - name: obol
     url: https://obolnetwork.github.io/helm-charts/
@@ -795,67 +815,24 @@ func generateValues(namespace, hostname, dashboardHostname, agentBaseURL, token,
             runAsGroup: %d
             fsGroup: %d
           initContainers:
+            # Single init container that runs the same Hermes image used by
+            # the runtime. The upstream image already ships /opt/hermes/.venv
+            # with all dependencies preinstalled, so there is no clone or pip
+            # install at pod start. fsGroup on the pod makes the PVC mount
+            # group-writable to the hermes user (uid 10000), so we don't need
+            # a recursive chown — which is the operation that fails on macOS
+            # virtiofs once the volume has read-only files like git pack
+            # files left from any previous install.
             - name: init-hermes-data
-              image: busybox:1.36
-              command:
-                - sh
-                - -c
-                - mkdir -p /data/.hermes && chown -R %d:%d /data/.hermes
-              securityContext:
-                runAsUser: 0
-              volumeMounts:
-                - name: data
-                  mountPath: /data
-            - name: bootstrap-hermes-install
               image: %s
               imagePullPolicy: IfNotPresent
               command:
                 - sh
                 - -ec
                 - |
-                  install_dir=%s
-                  repo_url=%s
                   mkdir -p /data/.hermes/home /data/.hermes/workspace
-                  lock_dir="${install_dir}.lock"
-                  got_lock=0
-                  for _ in $(seq 1 120); do
-                    if mkdir "$lock_dir" 2>/dev/null; then
-                      got_lock=1
-                      break
-                    fi
-                    sleep 1
-                  done
-                  if [ "$got_lock" != 1 ]; then
-                    echo "Timed out waiting for Hermes install lock: $lock_dir" >&2
-                    exit 1
-                  fi
-                  cleanup_lock() {
-                    rmdir "$lock_dir" 2>/dev/null || true
-                  }
-                  trap cleanup_lock EXIT
-
-                  if [ ! -d "$install_dir/.git" ] || { [ ! -f "$install_dir/pyproject.toml" ] && [ ! -f "$install_dir/setup.py" ]; }; then
-                    rm -rf "${install_dir}.tmp"
-                    if [ -e "$install_dir" ]; then
-                      mv "$install_dir" "${install_dir}.backup.$(date +%%s)"
-                    fi
-                    git clone --depth 1 "$repo_url" "${install_dir}.tmp"
-                    mv "${install_dir}.tmp" "$install_dir"
-                  fi
-                  cd "$install_dir"
-                  # Reinstall when the venv is missing the hermes binary OR
-                  # when the dashboard's web extra (fastapi/uvicorn) is absent.
-                  # The upstream image installs ".[all]" (which pulls in
-                  # ".[web]"); we re-create the venv from a fresh clone, so
-                  # the extras must be re-requested explicitly here.
-                  if [ ! -x "$install_dir/venv/bin/hermes" ] || \
-                     ! "$install_dir/venv/bin/python3" -c "import fastapi, uvicorn" >/dev/null 2>&1; then
-                    rm -rf "$install_dir/venv"
-                    uv venv --python python3 --system-site-packages venv
-                    VIRTUAL_ENV="$install_dir/venv" uv pip install -e ".[web]"
-                  fi
                   if [ -f /data/.hermes/state.db ]; then
-                    if ! python3 - <<'PY'
+                    if ! /opt/hermes/.venv/bin/python3 - <<'PY'
                   import sqlite3
                   conn = sqlite3.connect('/data/.hermes/state.db')
                   row = conn.execute('PRAGMA quick_check').fetchone()
@@ -871,8 +848,6 @@ func generateValues(namespace, hostname, dashboardHostname, agentBaseURL, token,
                       echo "Backed up malformed Hermes state DB to $backup_dir"
                     fi
                   fi
-                  cleanup_lock
-                  trap - EXIT
               volumeMounts:
                 - name: data
                   mountPath: /data
@@ -913,7 +888,7 @@ func generateValues(namespace, hostname, dashboardHostname, agentBaseURL, token,
                   value: %s
                 - name: OBOL_SKILLS_DIR
                   value: /data/.hermes/%s
-	`, desc.DataPVCName, namespace, desc.ServiceName, desc.ServiceName, namespace, desc.ServiceName, desc.ServiceName, desc.ServiceName, desc.ServiceName, containerUID, containerGID, containerGID, containerUID, containerGID, quoteYAML(image()), quoteYAML(hermesInstallDir), quoteYAML(hermesRepoURL), desc.ServiceName, quoteYAML(image()), quoteYAML(hermesBinary), desc.DefaultPort, desc.DefaultPort, quoteYAML(primary), quoteYAML(namespace), obolSkillsDirName)
+	`, desc.DataPVCName, namespace, desc.ServiceName, desc.ServiceName, namespace, desc.ServiceName, desc.ServiceName, desc.ServiceName, desc.ServiceName, containerUID, containerGID, containerGID, quoteYAML(image()), desc.ServiceName, quoteYAML(image()), quoteYAML(hermesBinary), desc.DefaultPort, desc.DefaultPort, quoteYAML(primary), quoteYAML(namespace), obolSkillsDirName)
 
 	if agentBaseURL != "" {
 		fmt.Fprintf(&b, "                - name: AGENT_BASE_URL\n                  value: %s\n", quoteYAML(agentBaseURL))

internal/hermes/hermes_test.go

@@ -143,18 +143,9 @@ func TestGenerateValues_UsesHermesNativeNames(t *testing.T) {
 		"containerPort: 8642",
 		"containerPort: 9119",
 		"init-hermes-data",
-		"bootstrap-hermes-install",
-		`install_dir="/data/.hermes/hermes-agent"`,
-		`repo_url="https://github.com/NousResearch/hermes-agent.git"`,
-		`lock_dir="${install_dir}.lock"`,
-		`Timed out waiting for Hermes install lock`,
-		`git clone --depth 1 "$repo_url" "${install_dir}.tmp"`,
-		"uv venv --python python3 --system-site-packages venv",
-		`uv pip install -e ".[web]"`,
-		`import fastapi, uvicorn`,
 		`PRAGMA quick_check`,
 		`state-db-corrupt-$ts`,
-		`- "/data/.hermes/hermes-agent/venv/bin/hermes"`,
+		`- "/opt/hermes/.venv/bin/hermes"`,
 		`- "hermes-obol-agent.obol.stack"`,
 		`- "obol-agent.obol.stack"`,
 		"name: hermes-dashboard",
@@ -165,6 +156,17 @@ func TestGenerateValues_UsesHermesNativeNames(t *testing.T) {
 		}
 	}
 
+	for _, banned := range []string{
+		"bootstrap-hermes-install",
+		"git clone",
+		"uv pip install",
+		"/data/.hermes/hermes-agent",
+	} {
+		if strings.Contains(values, banned) {
+			t.Fatalf("generateValues() should no longer reference %q (the in-pod git clone + venv build); use the upstream image's /opt/hermes/.venv instead:\n%s", banned, values)
+		}
+	}
+
 	var parsed any
 	if err := yaml.Unmarshal([]byte(values), &parsed); err != nil {
 		t.Fatalf("generateValues() produced invalid YAML: %v\n%s", err, values)
@@ -203,7 +205,7 @@ func TestHermesExecArgs_UsesNativeHermesBinary(t *testing.T) {
 		"-n", "hermes-obol-agent",
 		"deploy/hermes",
 		"--",
-		"/data/.hermes/hermes-agent/venv/bin/hermes",
+		"/opt/hermes/.venv/bin/hermes",
 		"skills",
 		"audit",
 	}