release-train: develop -> staging

.github/dependabot.yml

@@ -0,0 +1,28 @@
+# Dependabot config — security updates only.
+# `open-pull-requests-limit: 0` disables version updates; security updates have a separate
+# (built-in) limit of 10 and continue to fire when an advisory drops.
+# Labels and grouping defined here apply to security-update PRs.
+# Every group splits by update-type so a grouped PR never mixes patch with minor/major —
+# the auto-merge workflow gates on the PR-wide update-type (patch only).
+#
+# Mirrors the backend pilot (backend#577), which ran 2026-07-23 -> 2026-07-31 and produced
+# exactly one PR. Ecosystems and directories here come from a scan of this repo's tree.
+
+version: 2
+updates:
+  # GitHub Actions used in workflows
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 0
+    labels:
+      - "work-type:tech-debt"
+      - "dependencies"
+    groups:
+      actions-patches:
+        applies-to: security-updates
+        update-types: ["patch"]
+      actions-minor:
+        applies-to: security-updates
+        update-types: ["minor"]

.github/pull_request_template.md

@@ -2,7 +2,7 @@
 <!-- 1–3 sentences. What does this PR do and why? -->
 
 ## Related
-<!-- Same repo: Closes #123 · Cross-repo: Fixes tracebloc/backend#456 (owner-qualified — a bare backend#456 closes nothing). PRs land on develop, not the default branch, so confirm the issue actually closed. -->
+<!-- Same repo: Closes #123 · Cross-repo: Fixes tracebloc/client-runtime#456 (owner-qualified — a bare client-runtime#456 closes nothing). PRs land on develop, not the default branch, so closing keywords do not fire on merge — confirm the issue actually closed. -->
 
 ## Type of change
 - [ ] Feature
@@ -26,5 +26,6 @@
 - [ ] Docs updated if behavior or config changed
 - [ ] No secrets / credentials in the diff
 - [ ] For security-sensitive paths: appropriate reviewer requested
-- [ ] Terminal output follows [STYLE.md](../STYLE.md) — tone helpers (no hardcoded colour/emoji), "secure environment" not "workspace"; `bash scripts/check-style.sh` passes
 - [ ] Cross-repo issues use `Fixes tracebloc/<repo>#N` — a bare `repo#N` closes nothing
+- [ ] If this depends on a change in another repo: shipped **expand-then-contract** (additive first, consumers adopt later), or **Breaking change** ticked above with the rollout order in *Deployment notes* — repos promote independently, so the other change may not ship with this one
+- [ ] Terminal output follows [STYLE.md](../STYLE.md) — tone helpers (no hardcoded colour/emoji), "secure environment" not "workspace"; `bash scripts/check-style.sh` passes

.github/workflows/chart-version-guard.yml

@@ -0,0 +1,80 @@
+# Chart content can only reach installs via a NEW chart version — a Helm repo
+# publishes on version change, so an unbumped template/values edit reaches
+# nobody (this is exactly how the perIngestionTables flag block shipped to
+# staging but never rendered: PR #472 changed the template without bumping
+# Chart.yaml, so the published 1.9.7 stayed stale). This gate makes the bump
+# non-optional.
+name: Chart version guard
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  require-version-bump:
+    name: chart content ⇒ Chart.yaml version bump
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Require a Chart.yaml version bump when chart content changes
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          set -euo pipefail
+
+          # Fail CLOSED on an unusable base: without a diff we cannot know whether
+          # chart content changed, and "don't know" must never read as "nothing
+          # changed" — that is the same dark ship the gate exists to stop.
+          if [[ -z "${BASE_SHA:-}" ]]; then
+            echo "::error::Chart version guard could not determine the PR base SHA — refusing to report N/A without checking."
+            exit 1
+          fi
+          if ! changed="$(git diff --name-only "${BASE_SHA}...HEAD")"; then
+            echo "::error::Chart version guard could not diff ${BASE_SHA}...HEAD — refusing to report N/A without checking."
+            exit 1
+          fi
+
+          # Classify with bash builtins — NOT `printf … | grep -q`. Under the
+          # `set -o pipefail` above, `grep -q` closes the pipe on its FIRST match,
+          # so once the changed-file list passes the ~64KB pipe buffer `printf`
+          # takes SIGPIPE, the pipeline exits 141, and `if ! <pipeline>` reads a
+          # REAL client/templates/** change as "guard N/A" — silently skipping the
+          # bump check. Measured on ubuntu-24.04 (bash 5.2.21 / GNU grep 3.11):
+          # 65,622 bytes of paths already flips it. The mirror case is just as bad:
+          # a SIGPIPE on the `grep -qx 'client/Chart.yaml'` MATCH short-circuits the
+          # `&&` and fails a PR that did bump the version. No pipe here ⇒ neither is
+          # reachable, and no `|| true` (which would re-introduce a fail-open).
+          chart_content=0
+          chart_yaml=0
+          while IFS= read -r path; do
+            # Disjoint patterns, so one `case` covers both and always exits 0 —
+            # a `[[ … ]] && var=1` tail would itself trip `set -e` on a non-match.
+            case "$path" in
+              client/templates/*|client/values.yaml*) chart_content=1 ;;
+              client/Chart.yaml)                      chart_yaml=1 ;;
+            esac
+          done <<< "$changed"
+
+          if (( ! chart_content )); then
+            echo "No chart template/values change in this PR — guard N/A."
+            exit 0
+          fi
+          if (( chart_yaml )); then
+            if ! chart_diff="$(git diff "${BASE_SHA}...HEAD" -- client/Chart.yaml)"; then
+              echo "::error::Chart version guard could not diff client/Chart.yaml — refusing to pass without checking."
+              exit 1
+            fi
+            while IFS= read -r line; do
+              case "$line" in
+                '+version:'*)
+                  echo "Chart content changed and client/Chart.yaml 'version:' was bumped. ✓"
+                  exit 0 ;;
+              esac
+            done <<< "$chart_diff"
+          fi
+          echo "::error::client/templates/** or client/values.yaml changed, but client/Chart.yaml 'version:' was NOT bumped. A Helm chart repo publishes only on a version change, so an unbumped edit reaches no installs (this is how the perIngestionTables flag block shipped dark — PR #472). Bump client/Chart.yaml version in this PR."
+          exit 1

.github/workflows/code-quality-caller.yml

@@ -30,3 +30,4 @@ jobs:
       # fleet-wide + advisory soak done (backend#1303).
       soft-fail: false
       all-files: ${{ inputs.all-files || false }}
+      gitleaks-baseline: .gitleaks-baseline.json

.github/workflows/helm-ci.yaml

@@ -143,11 +143,27 @@ jobs:
             echo "OK — $label is multi-arch (amd64 + arm64)."
           }
 
-          # The floating tag is the default spawn target — always validate it.
-          if [ -z "$tag" ] || [ "$tag" = "null" ]; then
-            echo "::error::images.ingestor.tag is empty — the chart must define a floating tag to spawn by."; exit 1
+          # Every tag this chart can spawn by must be multi-arch. Since
+          # backend#1360 that is `tag` (an explicit override, empty by default)
+          # PLUS each per-environment entry in `channelTags` — an edge resolves
+          # exactly one of them, so a single-arch value in any of them breaks
+          # ingestion on arm64 for whichever environment lands on it.
+          checked=0
+          if [ -n "$tag" ] && [ "$tag" != "null" ]; then
+            assert_multiarch "${repo}:${tag}" "explicit tag override (images.ingestor.tag)"
+            checked=$((checked + 1))
+          fi
+          for env_key in dev stg prod; do
+            channel=$(yq ".images.ingestor.channelTags.${env_key}" client/values.yaml)
+            if [ -z "$channel" ] || [ "$channel" = "null" ]; then
+              continue
+            fi
+            assert_multiarch "${repo}:${channel}" "channelTags.${env_key}"
+            checked=$((checked + 1))
+          done
+          if [ "$checked" -eq 0 ]; then
+            echo "::error::no spawnable ingestor tag is defined — images.ingestor.tag is empty and channelTags has no entries, so an edge with no digest has nothing to spawn."; exit 1
           fi
-          assert_multiarch "${repo}:${tag}" "floating tag"
 
           # The prod pin is a chart DEFAULT that reaches every prod edge through
           # the fleet auto-upgrade, so a single-arch value here would break

.gitleaks-baseline.json

@@ -0,0 +1,44 @@
+[
+ {
+  "RuleID": "generic-api-key",
+  "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+  "StartLine": 66,
+  "EndLine": 66,
+  "StartColumn": 111,
+  "EndColumn": 171,
+  "Match": "SharedAccessKey=REDACTED;",
+  "Secret": "REDACTED",
+  "File": "docs/aks.md",
+  "SymlinkFile": "",
+  "Commit": "10477191a2f4e9f86f271747fa6628c401426a57",
+  "Link": "https://github.com/tracebloc/client/blob/10477191a2f4e9f86f271747fa6628c401426a57/docs/aks.md?plain=1#L66",
+  "Entropy": 4.788755,
+  "Author": "Asad Iqbal",
+  "Email": "asad.dsoft@gmail.com",
+  "Date": "2024-11-15T09:38:37Z",
+  "Message": "eks chart",
+  "Tags": [],
+  "Fingerprint": "10477191a2f4e9f86f271747fa6628c401426a57:docs/aks.md:generic-api-key:66"
+ },
+ {
+  "RuleID": "generic-api-key",
+  "Description": "Detected a Generic API Key, potentially exposing access to various services and sensitive operations.",
+  "StartLine": 67,
+  "EndLine": 67,
+  "StartColumn": 108,
+  "EndColumn": 207,
+  "Match": "AccountKey=REDACTED;",
+  "Secret": "REDACTED",
+  "File": "docs/aks.md",
+  "SymlinkFile": "",
+  "Commit": "10477191a2f4e9f86f271747fa6628c401426a57",
+  "Link": "https://github.com/tracebloc/client/blob/10477191a2f4e9f86f271747fa6628c401426a57/docs/aks.md?plain=1#L67",
+  "Entropy": 5.335638,
+  "Author": "Asad Iqbal",
+  "Email": "asad.dsoft@gmail.com",
+  "Date": "2024-11-15T09:38:37Z",
+  "Message": "eks chart",
+  "Tags": [],
+  "Fingerprint": "10477191a2f4e9f86f271747fa6628c401426a57:docs/aks.md:generic-api-key:67"
+ }
+]

client/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: client
 description: A unified Helm chart for tracebloc on AKS, EKS, bare-metal, and OpenShift
 type: application
-version: 1.9.8
-appVersion: "1.9.8"
+version: 1.9.9
+appVersion: "1.9.9"
 keywords:
   - tracebloc
   - kubernetes

client/templates/_helpers.tpl

@@ -274,6 +274,62 @@ defeating the pin. Every read is nil-guarded for the same reason.
 
 Usage: {{ include "tracebloc.ingestorDigest" . }}
 */}}
+{{/*
+  Resolved CLIENT_ENV, with the documented aliases normalized to the
+  canonical dev|stg|prod keys.
+
+  ONE definition on purpose. Bugbot caught the first cut normalizing inside
+  tracebloc.ingestorTag only, so CLIENT_ENV=production selected the prod
+  float tag while tracebloc.ingestorDigest still compared the RAW value to
+  "prod" and returned nothing -- silently dropping the reproducibility pin
+  (backend#1028/#1245) on an edge that looked correctly configured. Any future
+  consumer of CLIENT_ENV must go through here rather than re-deriving it, the
+  same reason ENV_ALIASES lives once in client-runtime proxy_config.
+*/}}
+{{- define "tracebloc.clientEnv" -}}
+{{- $raw := (default dict .Values.env).CLIENT_ENV | default "prod" -}}
+{{- $aliases := dict "development" "dev" "staging" "stg" "production" "prod" -}}
+{{- if hasKey $aliases $raw -}}
+{{- get $aliases $raw -}}
+{{- else -}}
+{{- $raw -}}
+{{- end -}}
+{{- end }}
+
+{{/*
+  Effective floating tag for spawned ingestion Jobs (backend#1360).
+
+  Precedence, mirroring tracebloc.ingestorDigest:
+    1. `images.ingestor.tag`          explicit override, any environment
+    2. `images.ingestor.channelTags[CLIENT_ENV]`   per-environment channel
+    3. "0.7"                          last-resort literal, so a release that
+                                      predates these keys still renders under
+                                      `--reuse-values`
+
+  Only consulted when no digest applies: jobs-manager builds `repo@digest`
+  when tracebloc.ingestorDigest is non-empty, and `repo:tag` otherwise
+  (client-runtime submit_ingestion_run._build_image_reference).
+
+  dev/stg resolve to the UNSIGNED internal channels. Prod is a semver float,
+  not a `:prod` tag — none is published.
+*/}}
+{{- define "tracebloc.ingestorTag" -}}
+{{- $ing := default dict .Values.images.ingestor -}}
+{{- $explicit := $ing.tag | default "" -}}
+{{- if $explicit -}}
+{{- $explicit -}}
+{{- else -}}
+{{- $clientEnv := include "tracebloc.clientEnv" . -}}
+{{- $channels := default dict $ing.channelTags -}}
+{{- $channel := get $channels $clientEnv | default "" -}}
+{{- if $channel -}}
+{{- $channel -}}
+{{- else -}}
+{{- "0.7" -}}
+{{- end -}}
+{{- end -}}
+{{- end }}
+
 {{- define "tracebloc.ingestorDigest" -}}
 {{- $ing := default dict .Values.images.ingestor -}}
 {{- $explicit := $ing.digest | default "" -}}
@@ -284,7 +340,7 @@ Usage: {{ include "tracebloc.ingestorDigest" . }}
 {{- if hasKey $ing "prodPin" -}}
 {{- $prodPin = $ing.prodPin -}}
 {{- end -}}
-{{- $clientEnv := (default dict .Values.env).CLIENT_ENV | default "prod" -}}
+{{- $clientEnv := include "tracebloc.clientEnv" . -}}
 {{- if and $prodPin (eq $clientEnv "prod") -}}
 {{- $ing.prodDigest | default "" -}}
 {{- end -}}

client/templates/jobs-manager-deployment.yaml

@@ -107,6 +107,24 @@ spec:
         - name: PER_INGESTION_TABLES
           value: "1"
         {{- end }}
+        {{- if .Values.perExperimentDbCreds }}
+        # RFC-0003 D10 (backend#1181): per-experiment MySQL credentials. When
+        # on, jobs-manager mints a short-lived MySQL user per experiment scoped
+        # to only its own table(s) and injects it into the training pod via a
+        # per-job Secret — instead of every pod sharing the root-equivalent
+        # edgeuser. TB_CREDMGR_* is the dedicated minting identity (provisioned
+        # by jobs-manager on startup from this generated Secret). Rendered only
+        # when enabled so default installs stay byte-identical.
+        - name: PER_EXPERIMENT_DB_CREDS
+          value: "1"
+        - name: TB_CREDMGR_USER
+          value: "tb_credmgr"
+        - name: TB_CREDMGR_PASSWORD
+          valueFrom:
+            secretKeyRef:
+              name: {{ include "tracebloc.secretName" . }}
+              key: TB_CREDMGR_PASSWORD
+        {{- end }}
         # Ingestor image wiring for the POST /internal/submit-ingestion-run
         # endpoint. jobs-manager spawns each ingestion Job from these values
         # (see client-runtime submit_ingestion_run._build_image_reference):
@@ -134,7 +152,7 @@ spec:
         - name: INGESTOR_IMAGE_REPOSITORY
           value: {{ (default dict .Values.images.ingestor).repository | default "ghcr.io/tracebloc/ingestor" | quote }}
         - name: INGESTOR_IMAGE_TAG
-          value: {{ (default dict .Values.images.ingestor).tag | default "0.7" | quote }}
+          value: {{ include "tracebloc.ingestorTag" . | quote }}
         - name: INGESTOR_IMAGE_DIGEST
           value: {{ include "tracebloc.ingestorDigest" . | quote }}
         - name: REQUESTS_PROXY_URL

client/templates/rbac.yaml

@@ -50,6 +50,15 @@ rules:
     # content matches before reusing it. Without `get`, those reads return
     # Forbidden and the endpoint 500s instead of the intended 409/replay.
     verbs: ["create", "get"]
+{{- if .Values.perExperimentDbCreds }}
+  # Per-experiment DB creds only: jobs-manager's revoke/sweep path deletes the
+  # per-job cred Secret. Scoped to `secrets` alone (never configmaps) and
+  # flag-gated so a default install grants no extra delete — least privilege,
+  # and byte-for-byte unchanged when off (Saqlain review).
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["delete"]
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
@@ -106,6 +115,15 @@ rules:
     # content matches before reusing it. Without `get`, those reads return
     # Forbidden and the endpoint 500s instead of the intended 409/replay.
     verbs: ["create", "get"]
+{{- if .Values.perExperimentDbCreds }}
+  # Per-experiment DB creds only: jobs-manager's revoke/sweep path deletes the
+  # per-job cred Secret. Scoped to `secrets` alone (never configmaps) and
+  # flag-gated so a default install grants no extra delete — least privilege,
+  # and byte-for-byte unchanged when off (Saqlain review).
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["delete"]
+{{- end }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding