fix(installer): stop k3s version pin from silently drifting (#547)

docs/INSTALL.md

@@ -279,6 +279,31 @@ With the client running, the typical follow-up is to land a dataset in the clust
 
 The chart **does not transport data into the cluster** — it points at data already accessible on the cluster's shared PVC (`client-pvc` by default, mounted at `/data/shared/` inside the ingestor Pod). Stage your CSV + image / text / annotation files there first; the ingestor chart README documents the `kubectl cp` pattern and production sync alternatives.
 
+### Staging on a local (hostpath) install
+
+On a laptop install (`hostPath.enabled: true`, the default for the OS installers) `/data/shared/` is just a folder on your machine — the dataset dir is `HOST_DATA_DIR\data\<dataset>` on Windows, `$HOST_DATA_DIR/data/<dataset>` on macOS/Linux — so you stage a dataset by copying it there directly (no `kubectl cp` needed).
+
+Use **idempotent** commands so re-running the step (for example after the installer got re-run) never errors on an already-staged dataset:
+
+**Windows (PowerShell)** — `New-Item -Force` doesn't fail when the folder already exists, and `robocopy` merges into an existing target instead of throwing `already exists`:
+
+```powershell
+$dst = "$env:USERPROFILE\.tracebloc\data\shapes-demo"    # HOST_DATA_DIR\data\<dataset>
+New-Item -ItemType Directory -Force -Path $dst | Out-Null
+robocopy "$env:USERPROFILE\Downloads\sample_dataset\shapes-demo" $dst /E
+# robocopy exit codes 0-7 are success (>=8 is a real error); safe to re-run.
+```
+
+> Plain `mkdir` and `Copy-Item -Recurse` are **not** idempotent — a second run throws `An item with the specified name … already exists` even though the earlier copy succeeded. That error is harmless (the data is already staged), but the commands above avoid it.
+
+**macOS / Linux** — `mkdir -p` and `cp -R` (or `rsync -a`) are already idempotent:
+
+```bash
+dst="$HOME/.tracebloc/data/shapes-demo"                  # $HOST_DATA_DIR/data/<dataset>
+mkdir -p "$dst"
+cp -R ~/Downloads/sample_dataset/shapes-demo/. "$dst/"
+```
+
 Example: once you've staged a cats-vs-dogs image classification dataset under `/data/shared/cats-dogs/` on the PVC, the `ingest.yaml` describes what's there:
 
 ```yaml

scripts/check-facts.sh

@@ -26,6 +26,7 @@ SPEC="scripts/spec/facts.env"
 COMMON="scripts/lib/common.sh"
 SUMMARY="scripts/lib/summary.sh"
 PS1="scripts/install-k8s.ps1"
+CLUSTER="scripts/lib/cluster.sh"
 
 MODE="write"
 case "${1:-}" in
@@ -125,12 +126,47 @@ while [ "$i" -lt "${#FACT_NAMES[@]}" ]; do
   i=$(( i + 1 ))
 done
 
+# Structural guard (#547 / F4): the fact table above only compares the pinned
+# VERSION STRINGS — it does NOT verify the create command actually WIRES the k3s
+# pin into the cluster. #547 drifted precisely because `--image rancher/k3s:<ver>`
+# can be dropped/gated while the version string stays correct and CI stays green.
+# Assert the create-time wiring is present in both installers so a refactor can't
+# silently unpin k3s. Fixed-string (grep -F): these are literal shell/PS tokens.
+_check_wiring() {  # name  file  literal
+  if [[ ! -f "$2" ]]; then
+    echo "  ✖ ${1}: ${2} not found" >&2; return 1
+  fi
+  if grep -qF "$3" "$2"; then
+    echo "  ✔ ${1}: k3s --image pin present in ${2}"; return 0
+  fi
+  echo "  ✖ ${1}: create-time '${3}' not found in ${2} — k3s could float (#547)" >&2; return 1
+}
+# Wiring failures are tracked SEPARATELY from version drift: `--write` restamps
+# version strings but CANNOT restore create-time wiring, so a wiring gap must not
+# emit the "run --write" hint (Bugbot #565) — it needs a hand-fix.
+wiring_fail=0
+if [[ "$MODE" == "check" ]]; then
+  _check_wiring "cluster.sh:k3s-image-pin"      "$CLUSTER" 'rancher/k3s:${K8S_VERSION}' || wiring_fail=$(( wiring_fail + 1 ))
+  _check_wiring "install-k8s.ps1:k3s-image-pin" "$PS1"     'rancher/k3s:$K8S_VERSION'    || wiring_fail=$(( wiring_fail + 1 ))
+fi
+
 if [[ "$MODE" == "check" ]]; then
+  rc=0
   if [[ "$drift" -ne 0 ]]; then
     echo "" >&2
     echo "check-facts: ${drift} fact(s) drifted from ${SPEC}. Run 'scripts/check-facts.sh --write' and commit." >&2
-    exit 1
+    rc=1
+  fi
+  if [[ "$wiring_fail" -ne 0 ]]; then
+    echo "" >&2
+    echo "check-facts: the k3s --image pin is missing from the create path in ${wiring_fail} file(s) (see ✖ above)." >&2
+    echo "check-facts: this is a WIRING gap, not a version bump — '--write' cannot fix it. Restore the create-time" >&2
+    echo "             --image k3s pin by hand using the EXACT literal each ✖ line above shows — the two shells differ:" >&2
+    echo "             bash cluster.sh uses 'rancher/k3s:\${K8S_VERSION}' (braces); PowerShell install-k8s.ps1 uses" >&2
+    echo "             'rancher/k3s:\$K8S_VERSION' (no braces). So k3s can't float (#547)." >&2
+    rc=1
   fi
+  [[ "$rc" -eq 0 ]] || exit 1
   echo "check-facts: all installer facts match ${SPEC}."
 else
   [[ "$drift" -eq 0 ]] || { echo "check-facts: ${drift} consumer(s) could not be stamped (see above)." >&2; exit 1; }

scripts/install-k8s.ps1

@@ -17,7 +17,7 @@
 #    $env:CLUSTER_NAME  = "myapp"          default: tracebloc
 #    $env:SERVERS       = "1"              default: 1  (control-plane nodes)
 #    $env:AGENTS        = "1"              default: 1  (worker nodes)
-#    $env:K8S_VERSION   = "v1.29.4-k3s1"  default: latest
+#    $env:K8S_VERSION   = "v1.29.4-k3s1"  default: v1.29.4-k3s1 (pinned + validated; "latest" is UNSUPPORTED — see #547)
 #    $env:HOST_DATA_DIR = "C:\data"        default: $env:USERPROFILE\.tracebloc (LOCAL disk; no NFS/UNC)
 #    $env:CLIENT_ENV    = "dev"            optional; if not set, CLIENT_ENV is not added to env in values
 #    $env:TRACEBLOC_TRAINING_RESOURCES = "cpu=4,memory=16Gi"   optional; overrides the machine-sized training default
@@ -2201,6 +2201,42 @@ function Write-HostCaCreateHint {
   Write-Host ""
 }
 
+# Warn (never fatal) when the RUNNING cluster's k3s differs from the validated pin.
+# k3s is baked in at create time; a cluster born unpinned, on an older installer, or
+# with K8S_VERSION=latest keeps its version across later pinned re-runs -- the #547
+# incident (a client ran k3s v1.35.5 while the pin was v1.29.4-k3s1). Called from
+# BOTH the reuse path in New-K3dCluster AND the completed+healthy fast-path in main
+# (Bugbot #565), so a healthy-but-drifted cluster still gets the recreate guidance.
+# Silent no-op if the image can't be read or isn't a parseable rancher/k3s:<tag>
+# (e.g. a digest-only pin) -- never false-warn.
+function Test-K3sVersionDrift {
+  if ($K8S_VERSION -eq "" -or $K8S_VERSION -eq "latest") { return }
+  # Bounded (installer rule: every docker probe must have a deadline) so a wedged
+  # Docker engine can't hang the "already healthy" fast-path after success prints
+  # (#565 Bugbot). Mirrors Test-ClusterRunning's Start-Job + timeout pattern.
+  $k3sImage = ""
+  $job = Start-Job -InitializationScript $JobInit -ScriptBlock {
+    param($n) (docker inspect "k3d-$n-server-0" --format '{{.Config.Image}}' 2>$null | Out-String)
+  } -ArgumentList $CLUSTER_NAME
+  if (Wait-JobWithProgress -Job $job -TimeoutSec 15 -Message "Checking k3s version") {
+    $k3sImage = (Receive-Job $job -ErrorAction SilentlyContinue | Out-String).Trim()
+  } else {
+    Log "docker inspect (k3s version) timed out; skipping the version-drift check."
+  }
+  Remove-Job $job -Force -ErrorAction SilentlyContinue
+  if ($k3sImage -match 'rancher/k3s:([^@\s]+)') {
+    $runningK3s = $Matches[1]
+    if ($runningK3s -ne $K8S_VERSION) {
+      Warn "The existing '$CLUSTER_NAME' cluster runs k3s '$runningK3s', not the validated pin '$K8S_VERSION'."
+      Hint "k3s version is fixed when the cluster is created -- it can't be changed on a running cluster."
+      Hint "This cluster was created by an older/unpinned installer or with K8S_VERSION=latest (#547). To move"
+      Hint "onto the validated version, recreate it:"
+      Hint "  k3d cluster delete $CLUSTER_NAME  (then re-run this installer)."
+      Hint "  (data under HOST_DATA_DIR is kept; recreate rebinds it.)"
+    }
+  }
+}
+
 function New-K3dCluster {
   Log "Creating k3d cluster: '$CLUSTER_NAME'"
 
@@ -2296,6 +2332,11 @@ function New-K3dCluster {
         Err "Existing cluster is missing the dataset bind mount - refusing to install datasets onto ephemeral storage."
       }
     }
+
+    # k3s version drift: a cluster born unpinned/old/latest keeps its k3s across
+    # pinned re-runs (#547). Shared with the completed+healthy fast-path in main so
+    # a healthy-but-drifted cluster is warned too (Bugbot #565).
+    Test-K3sVersionDrift
   } else {
     # Creating a FRESH cluster — never silently adopt data an earlier install
     # left under HOST_DATA_DIR (RFC-0003 §4 / #376; parity with the bash guard).
@@ -2328,7 +2369,18 @@ function New-K3dCluster {
     # local /tracebloc tree. No-op when unset.
     if ($HOST_DATASET_DIR) { $k3dArgs += @("-v", "${HOST_DATASET_DIR}:/tracebloc-data@all") }
 
-    if ($K8S_VERSION -ne "" -and $K8S_VERSION -ne "latest") { $k3dArgs += @("--image", "rancher/k3s:$K8S_VERSION") }
+    # Pin k3s at create time (#547). $K8S_VERSION defaults to the validated pin, so
+    # a normal install ALWAYS passes --image; the version is baked into the node
+    # image and can't change later. K8S_VERSION=latest is an unsupported opt-out
+    # that floats to k3d's own bundled default (how a client landed on v1.35.5) —
+    # honour it but warn loudly.
+    if ($K8S_VERSION -eq "latest") {
+      Warn "K8S_VERSION=latest runs an UNVALIDATED k3s (k3d's bundled default), not the tested pin."
+      Hint "The chart is validated against a specific k3s release; 'latest' is unsupported and has stranded installs (#547)."
+      Hint "Unset K8S_VERSION (or pin it to a validated tag) to use the tested version."
+    } elseif ($K8S_VERSION -ne "") {
+      $k3dArgs += @("--image", "rancher/k3s:$K8S_VERSION")
+    }
     if ($K3D_GPU_FLAG -ne "") {
       $k3dArgs += $K3D_GPU_FLAG
       Log "GPU flag active: $K3D_GPU_FLAG"
@@ -4168,6 +4220,10 @@ Print-Roadmap
 # finish the interrupted walk).
 if ((-not $Resume) -and $script:InstallState.completed -and (Test-ToolsPresent) -and (Test-ClusterRunning) -and (Test-ClientHealthy)) {
   Ok "tracebloc is already installed and the client is healthy -- nothing to do."
+  # A healthy cluster can still be running a DRIFTED k3s (the #547 steady state);
+  # this fast-path exits before New-K3dCluster's reuse check, so warn here too
+  # (Bugbot #565). Non-fatal: the client is healthy, we just flag the version.
+  Test-K3sVersionDrift
   Hint "Delete $(Get-InstallStatePath) (or set a fresh HOST_DATA_DIR) to force a full reinstall."
   Unregister-ResumeAfterReboot
   try { Stop-Transcript | Out-Null } catch {}

scripts/install-k8s.sh

@@ -19,7 +19,7 @@
 #                                not prompted — the client is identified by its credentials)
 #    SERVERS=1                   default: 1  (control-plane nodes)
 #    AGENTS=1                    default: 1  (worker nodes)
-#    K8S_VERSION=v1.29.4-k3s1   default: latest stable k3s
+#    K8S_VERSION=v1.29.4-k3s1   default: v1.29.4-k3s1 (pinned + validated; "latest" is UNSUPPORTED — see #547)
 #    K3D_VERSION=v5.9.0          default: v5.9.0  (k3d release tag; "latest" resolves at install time)
 #    HOST_DATA_DIR=~/.tracebloc  default: ~/.tracebloc
 #    TB_STORAGE_MODE=node-local  default: hostpath  (RFC-0003 Option C, flag-gated)

scripts/lib/assess.sh

@@ -207,6 +207,12 @@ assess_existing_install() {
   case "$INSTALL_STATE" in
     healthy)
       echo ""
+      # A healthy cluster can still be running a DRIFTED k3s (born unpinned, on an
+      # older installer, or with K8S_VERSION=latest) — the #547 STEADY STATE. This
+      # fast-path hands off and exits before _handle_existing_cluster, so its
+      # reuse-path drift check never runs; surface the warning here too so a
+      # healthy-but-drifted client still sees the recreate guidance (Bugbot #565).
+      declare -F _check_existing_cluster_k8s_version >/dev/null 2>&1 && _check_existing_cluster_k8s_version
       _assess_handoff        # prints the "already set up" line, runs `tracebloc`, exit 0
       ;;
     degraded)

scripts/lib/cluster.sh

@@ -574,6 +574,45 @@ _handle_existing_cluster() {
   _check_existing_cluster_bind
   _check_existing_cluster_dataset_mount
   _check_existing_cluster_storage_mode
+  _check_existing_cluster_k8s_version
+}
+
+# k3s version is fixed when the cluster is created (baked into the node image);
+# it can't be changed on a running cluster. A cluster created by an older/unpinned
+# installer or with K8S_VERSION=latest keeps whatever k3s it was born with, EVEN
+# ACROSS later correctly-pinned re-runs — the single best explanation for the #547
+# incident, where a client ran k3s v1.35.5 while the pin was v1.29.4-k3s1 and every
+# re-run silently reused the drifted cluster. Warn on drift with the recreate
+# remedy so it's surfaced instead of reused. Silent no-op if Docker is down, the
+# server can't be inspected, or the image isn't a parseable rancher/k3s:<tag>
+# (e.g. a digest-only pin) — never false-warn.
+_check_existing_cluster_k8s_version() {
+  [[ -z "${K8S_VERSION:-}" || "$K8S_VERSION" == "latest" ]] && return 0
+  local server_container="k3d-${CLUSTER_NAME}-server-0"
+  local image
+  # Bounded (installer rule: every docker/kubectl probe must have a deadline): both
+  # healthy fast-paths call this, so a wedged Docker engine must not hang an
+  # "already healthy" re-run after success is printed (#565 Bugbot). 124 on timeout
+  # → the `|| return 0` makes it a silent no-op, same as an inspect failure.
+  image=$(_bounded "${TB_DOCKER_INSPECT_TIMEOUT:-10}" docker inspect "$server_container" --format '{{.Config.Image}}' 2>/dev/null) || return 0
+  [[ -z "$image" ]] && return 0
+  case "$image" in
+    *rancher/k3s:*) : ;;
+    *) return 0 ;;   # unexpected image ref — don't guess
+  esac
+  local running="${image##*rancher/k3s:}"   # strip up to the tag
+  running="${running%%@*}"                   # drop any @sha256:... digest suffix
+  [[ -z "$running" ]] && return 0
+  if [[ "$running" != "$K8S_VERSION" ]]; then
+    echo ""
+    warn "The existing '$CLUSTER_NAME' cluster runs k3s '$running', not the validated pin '$K8S_VERSION'."
+    hint "k3s version is fixed when the cluster is created — it can't be changed on a running cluster."
+    hint "This cluster was created by an older/unpinned installer or with K8S_VERSION=latest (#547). To move"
+    hint "onto the validated version, recreate it:"
+    hint "  k3d cluster delete $CLUSTER_NAME  &&  re-run this installer."
+    hint "  (hostpath mode keeps your data under ${HOST_DATA_DIR:-your data dir}; node-local mode loses in-cluster data on recreate.)"
+    echo ""
+  fi
 }
 
 # k3d bakes proxy env into containers at create time; it cannot be added to a
@@ -807,7 +846,20 @@ _create_new_cluster() {
   # while mysql + logs stay on the local /tracebloc tree. No-op when unset.
   [[ -n "${HOST_DATASET_DIR:-}" ]] && K3D_ARGS+=(-v "${HOST_DATASET_DIR}:/tracebloc-data@all")
 
-  [[ -n "$K8S_VERSION" && "$K8S_VERSION" != "latest" ]] && K3D_ARGS+=(--image "rancher/k3s:${K8S_VERSION}")
+  # Pin k3s at create time. common.sh defaults K8S_VERSION to the validated pin,
+  # so a normal install ALWAYS passes --image; the version is fixed into the node
+  # image and can't be changed later. An explicit K8S_VERSION=latest is an
+  # unsupported opt-out that floats to k3d's OWN bundled default k3s — the exact
+  # drift that stranded a client on v1.35.5 while the pin was v1.29.4 (#547) — so
+  # honour it but warn loudly. (Empty only happens when cluster.sh is sourced
+  # without common.sh, e.g. the unit harness; leave it a no-op there.)
+  if [[ "$K8S_VERSION" == "latest" ]]; then
+    warn "K8S_VERSION=latest runs an UNVALIDATED k3s (k3d's bundled default), not the tested pin."
+    hint "The chart is validated against a specific k3s release; 'latest' is unsupported and has stranded installs (#547)."
+    hint "Unset K8S_VERSION (or pin it to a validated tag) to use the tested version."
+  elif [[ -n "$K8S_VERSION" ]]; then
+    K3D_ARGS+=(--image "rancher/k3s:${K8S_VERSION}")
+  fi
 
   if [[ ${#K3D_GPU_FLAGS[@]} -gt 0 ]]; then
     K3D_ARGS+=("${K3D_GPU_FLAGS[@]}")

scripts/manifest.sha256

@@ -1,18 +1,18 @@
-6a5a3fda82b12ae43e9c78d9f84b273c3d2f8a339c7c07767a066ac23121db07  scripts/install-k8s.sh
+8e256be14eff4b50e088a54062356110e50fbcafb583849242f2a0de086e29c3  scripts/install-k8s.sh
 1bc9e544b56b54ce346a9c6c3a7e8781169a70c50ac5877732587b991e85075d  scripts/lib/common.sh
 99e8149769cb08d9a26d35aa9714c33b23380951bbaab682c8dcb6e277e93bb7  scripts/lib/preflight.sh
 19be2771df0e1a41b4fa9678e1cf6a77492304f66f73cf705a2ae42b1dac2ba3  scripts/lib/detect-gpu.sh
 d8c29bc8bd1f4633300940894da0f6527ca0a1dd7a3cfcbc80aad19dfd4d88cb  scripts/lib/gpu-nvidia.sh
 b569eec2d8ffb9673da287a2a59d249a7dbc7236c98ab6a5062136bcc69a942c  scripts/lib/gpu-amd.sh
 fc3dacf419b66373a7e1b7c3ce0f44ec47fdfa3f7039cb2697ac2dd065344596  scripts/lib/setup-macos.sh
 f8f398191e03d750f61eca4e867b4f8ccb8b447c34c9803daa0ad4f3a49701aa  scripts/lib/setup-linux.sh
-1bdf2bf09c07f096551a9405af232f19f11649064f9e5123b4eca0f9f55ab20d  scripts/lib/cluster.sh
+b7b19dea83b6ee988a563105082c233264b7a3c35b82753405a4d76f011548d2  scripts/lib/cluster.sh
 045caf6efeb583e5005d881d9281edae6a6aedf8f318b0a4021964b3b4b29cc5  scripts/lib/gpu-plugins.sh
 f902abc5a9f2f65467b1a324804b5205b6a30fa246b4bd8328066d79595ffb05  scripts/lib/install-client-helm.sh
 61c1c887d158af52d4da4734b3bfa83205b2600ae7a291bfb3074daf3d9ffb55  scripts/lib/install-cli.sh
 725a85e4927761d8362221012ad1b69b380e36ec7fcfcf4c04b801f0994bce5c  scripts/lib/provision.sh
-e2ea63d844e6649f1d3aaae9fd4733845a1a39df37d68abbaeda00330f9e1c7e  scripts/lib/assess.sh
+e373403d7bb5ce3728b8d21af89e6bf672cc35bbf8938541eb527ae19cb9473b  scripts/lib/assess.sh
 911fd0714b17357bb205fc8a8fa8e13eedc1a9632a2f63d4ead9f8d8c7ee546f  scripts/lib/probe.sh
 38761a6c56dc85b3f5742df036e6a2ec2baa0adb0c90b3753b6706779528b7be  scripts/lib/summary.sh
 77e03332ebfab1ef759c6148a57afcf479c02c5dc6cc7b0e0e680f58e20cd364  scripts/lib/diagnose.sh
-d51d4e83ebc9adb1e2c84f2433f10a6c08a8741081d513e273448520c8d289bb  scripts/install-k8s.ps1
+50f382bb1503af4167cbd5738b3d6755be1ca4096fcd14fc6286db3f5100c28e  scripts/install-k8s.ps1

scripts/tests/assess.bats

@@ -364,3 +364,26 @@ _depname() {
   refute_has "first time" "$output"            # no ceremony when a cluster already exists
   refute_has "HOME_SCREEN" "$output"
 }
+
+# ── healthy fast-path still surfaces k3s drift (#547, Bugbot #565) ───────────
+# The reuse-path drift check in _handle_existing_cluster is never reached when a
+# re-run classifies as healthy (it hands off + exits), so assess_existing_install
+# must run the check itself before the handoff — else a healthy-but-drifted
+# cluster (a client already up on a floated k3s) is silently reused.
+@test "assess_existing_install: healthy branch runs the k3s drift check before handoff" {
+  _assess_classify() { INSTALL_STATE=healthy; INSTALL_STATE_REASON=""; }
+  _check_existing_cluster_k8s_version() { echo "DRIFT_CHECK_RAN"; }
+  _assess_handoff() { echo "HANDOFF_RAN"; }        # stub: don't exit under `run`
+  run assess_existing_install
+  [ "$status" -eq 0 ]
+  assert_has "DRIFT_CHECK_RAN" "$output"
+  assert_has "HANDOFF_RAN" "$output"
+}
+
+@test "assess_existing_install: --force bypass skips the drift check entirely" {
+  export TB_FORCE_REINSTALL=1
+  _check_existing_cluster_k8s_version() { echo "DRIFT_CHECK_RAN"; }
+  run assess_existing_install
+  [ "$status" -eq 0 ]
+  refute_has "DRIFT_CHECK_RAN" "$output"
+}