fix(slab): the guard watches memcg-attributed truth, not the drifting counter

cmd/gha-fleet/main.go

@@ -1816,12 +1816,20 @@ func runSlabHeal(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "gha-fleet: slab-heal: %v\n", err)
 		return 1
 	}
-	sunreclaim, err := slabheal.ParseSUnreclaim(meminfo)
+	counter, err := slabheal.ParseSUnreclaim(meminfo)
 	meminfo.Close()
 	if err != nil {
 		fmt.Fprintf(stderr, "gha-fleet: slab-heal: %v\n", err)
 		return 1
 	}
+	attributed, attributedOK := uint64(0), false
+	if stat, statErr := os.Open("/sys/fs/cgroup/memory.stat"); statErr == nil {
+		if value, parseErr := slabheal.ParseAttributedSlabUnreclaimable(stat); parseErr == nil {
+			attributed, attributedOK = value, true
+		}
+		stat.Close()
+	}
+	sunreclaim, slabSource := slabheal.PreferAttributed(attributed, attributedOK, counter)
 	members, err := client.GetClusterMembers()
 	if err != nil {
 		fmt.Fprintf(stderr, "gha-fleet: slab-heal: read cluster members: %v\n", err)
@@ -1870,7 +1878,7 @@ func runSlabHeal(args []string, stdout, stderr io.Writer) int {
 			"reason": decision.Reason, "sunreclaim_bytes": sunreclaim,
 		})
 	}
-	reason := fmt.Sprintf("%sSUnreclaim %.1f GiB over the %.1f GiB budget", slabheal.HealReasonPrefix, float64(sunreclaim)/(1<<30), float64(*thresholdBytes)/(1<<30))
+	reason := fmt.Sprintf("%sSUnreclaim %.1f GiB (%s) over the %.1f GiB budget", slabheal.HealReasonPrefix, float64(sunreclaim)/(1<<30), slabSource, float64(*thresholdBytes)/(1<<30))
 	result, err := memberdrain.Drain(ctx, deps, memberdrain.Options{
 		MemberName: cfg.Incus.Cluster.MemberName, Reason: reason,
 		TimerUnit: *timerUnit, Timeout: *timeout, Poll: *poll, Apply: true,

config/observability-rules.yaml

@@ -364,7 +364,7 @@ rules:
     severity: ticket
     query_language: promql
     stream_name: system_memory_usage
-    expression: max(system_memory_usage{service_namespace="nddev-github-actions",state="slab_unreclaimable"})
+    expression: max(gha_fleet_host_slab_unreclaimable_attributed_bytes)
     operator: ">"
     threshold: 2147483648
     evaluation_seconds: 300
@@ -380,9 +380,9 @@ rules:
     enabled: true
     owner: fleet-operations
     runbook: https://github.com/NDDev-OpenNetwork/github-actions/blob/main/docs/runbooks/fleet-alerts.md
-    summary: Unreclaimable kernel slab exceeds two GiB on a fleet host.
-    action: Preserve slab, audit and AppArmor evidence; close member admission and roll to the fixed kernel after jobs drain.
-    recovery: Every fleet host remains below two GiB unreclaimable slab after workload churn.
+    summary: Unreclaimable kernel slab exceeds two GiB on a fleet host, by memcg-attributed measurement.
+    action: This watches the root cgroup's attributed slab, not the drifting global counter (github-actions#264 -- the counter overstated truth by ~1 GiB under container churn). A real breach means live kernel objects; preserve slab, audit and AppArmor evidence, then let gha-slab-heal drain and reboot the member.
+    recovery: Every fleet host remains below two GiB attributed unreclaimable slab after workload churn.
   - id: kernel_workqueue_hog
     severity: ticket
     query_language: promql

docs/maintenance-windows.md

@@ -61,7 +61,15 @@ socket every ~11 seconds. Two consequences, both observed:
 ## The slab healer
 
 `gha-slab-heal.timer` (hourly, per member) runs `gha-fleet slab-heal --apply`:
-heal only above the alert's own SUnreclaim budget, only with an open gate
+heal only above the alert's own SUnreclaim budget — measured as the root
+cgroup's **memcg-attributed** `slab_unreclaimable` when cgroup v2 exposes
+it, because the global `/proc/meminfo` counter drifts under container churn
+(a member showed 1041 MiB there against 44 MiB of attributed truth on
+2026-09-01, the gap unattributable to any live cache and immune to
+drop_caches and slab shrink; github-actions#264). Both views ship as host
+gauges (`…slab_unreclaimable_counter_bytes` / `…_attributed_bytes`) and the
+`kernel_slab_unreclaimable` alert watches the attributed one. Then: only
+with an open gate
 (a closed gate belongs to its operator), only when every other member is
 open, only on a jobless member, only outside a twelve-hour cooldown. The heal
 drains through the marker, records the cooldown, and reboots only after the

internal/observabilityrules/rules_test.go

@@ -30,7 +30,7 @@ func TestRepositoryRulesUseCurrentMetricSemantics(t *testing.T) {
 		"compute_pressure_observer_missing": `count(up{service_name="pressure-state"} == 1)`,
 		"compute_pressure_state_stale":      "gha_fleet_pressure_observer_up",
 		"compute_root_disk_low":             "system_filesystem_usage",
-		"kernel_slab_unreclaimable":         `state="slab_unreclaimable"`,
+		"kernel_slab_unreclaimable":         "gha_fleet_host_slab_unreclaimable_attributed_bytes",
 		"audit_suppression_burst":           `signal_class="audit_suppressed"`,
 		"kernel_workqueue_hog":              `signal_class="kernel_workqueue_hog"`,
 		"host_compliance_observer_missing":  "gha_fleet_host_compliance_observer_up",

internal/pressureobserve/compliance.go

@@ -24,13 +24,34 @@ type Compliance struct {
 	StandardUpdatesAvailable    int
 	ESMSecurityUpdatesAvailable int
 	PackageInventoryAgeSeconds  float64
+	// Two views of unreclaimable slab, exported side by side because the
+	// global /proc/meminfo counter drifts: on 2026-09-01 a member reported
+	// 1041 MiB there while the memcg-attributed truth was 44 MiB, the gap
+	// unattributable to any live cache and immune to drop_caches and slab
+	// shrink -- a kernel accounting leak under container churn, not memory.
+	// -1 means the reading was unavailable.
+	SlabUnreclaimableCounterBytes    float64
+	SlabUnreclaimableAttributedBytes float64
 }
 
 func CollectCompliance(root string, now time.Time) Compliance {
 	if root == "" {
 		root = "/"
 	}
-	result := Compliance{PackageInventoryAgeSeconds: -1, SRSOStatus: "unknown"}
+	result := Compliance{
+		PackageInventoryAgeSeconds: -1, SRSOStatus: "unknown",
+		SlabUnreclaimableCounterBytes: -1, SlabUnreclaimableAttributedBytes: -1,
+	}
+	if meminfo, err := os.ReadFile(filepath.Join(root, "proc", "meminfo")); err == nil {
+		if value, ok := memoryField(string(meminfo), "SUnreclaim:", 1024); ok {
+			result.SlabUnreclaimableCounterBytes = value
+		}
+	}
+	if stat, err := os.ReadFile(filepath.Join(root, "sys", "fs", "cgroup", "memory.stat")); err == nil {
+		if value, ok := memoryField(string(stat), "slab_unreclaimable", 1); ok {
+			result.SlabUnreclaimableAttributedBytes = value
+		}
+	}
 	result.RebootRequired = regularFileExists(filepath.Join(root, "var", "run", "reboot-required"))
 
 	kernel, kernelErr := os.ReadFile(filepath.Join(root, "proc", "sys", "kernel", "osrelease"))
@@ -85,11 +106,30 @@ func RenderCompliance(state Compliance) string {
 	gauge("gha_fleet_host_standard_updates_available", "Standard Ubuntu package updates currently available.", float64(state.StandardUpdatesAvailable))
 	gauge("gha_fleet_host_esm_security_updates_available", "Additional security updates available only through Ubuntu ESM Apps.", float64(state.ESMSecurityUpdatesAvailable))
 	gauge("gha_fleet_host_package_inventory_age_seconds", "Age of the update-notifier package inventory, or -1 when unavailable.", state.PackageInventoryAgeSeconds)
+	gauge("gha_fleet_host_slab_unreclaimable_counter_bytes", "Unreclaimable slab as the drifting global /proc/meminfo counter reports it, or -1 when unavailable.", state.SlabUnreclaimableCounterBytes)
+	gauge("gha_fleet_host_slab_unreclaimable_attributed_bytes", "Unreclaimable slab as the root cgroup memory.stat attributes it -- the truthful view, or -1 when unavailable.", state.SlabUnreclaimableAttributedBytes)
 	fmt.Fprintf(&output, "# HELP gha_fleet_host_kernel_info Running host kernel identity.\n# TYPE gha_fleet_host_kernel_info gauge\ngha_fleet_host_kernel_info{release=%q} 1\n", escapeLabel(state.KernelRelease))
 	fmt.Fprintf(&output, "# HELP gha_fleet_host_srso_status Speculative return stack overflow status reported by the running kernel.\n# TYPE gha_fleet_host_srso_status gauge\ngha_fleet_host_srso_status{status=%q} 1\n", escapeLabel(state.SRSOStatus))
 	return output.String()
 }
 
+// memoryField finds "<key> <number>" in meminfo/memory.stat content and
+// scales it to bytes (meminfo counts KiB, memory.stat counts bytes).
+func memoryField(content, key string, scale float64) (float64, bool) {
+	for _, line := range strings.Split(content, "\n") {
+		fields := strings.Fields(line)
+		if len(fields) < 2 || fields[0] != key {
+			continue
+		}
+		value, err := strconv.ParseFloat(fields[1], 64)
+		if err != nil {
+			return 0, false
+		}
+		return value * scale, true
+	}
+	return 0, false
+}
+
 func classifySRSO(value string) string {
 	normalized := strings.ToLower(strings.TrimSpace(value))
 	switch {

internal/pressureobserve/compliance_test.go

@@ -24,6 +24,8 @@ func TestCollectAndRenderCompliance(t *testing.T) {
 	write("sys/devices/system/cpu/vulnerabilities/spec_rstack_overflow", "Vulnerable: Safe RET, no microcode\n")
 	write("var/lib/update-notifier/updates-available", "0 updates can be applied immediately.\n3 additional security updates can be applied with ESM Apps.\n")
 	write("var/run/reboot-required", "*** System restart required ***\n")
+	write("proc/meminfo", "MemTotal: 16000000 kB\nSUnreclaim:     1066110 kB\nSReclaimable: 200000 kB\n")
+	write("sys/fs/cgroup/memory.stat", "anon 1\nslab_reclaimable 766643200\nslab_unreclaimable 46323712\nslab 812966912\n")
 	now := time.Now().UTC()
 	state := CollectCompliance(root, now)
 	if !state.Complete || !state.RebootRequired || state.KernelRelease != "6.8.0-138-generic" || state.SRSOStatus != "vulnerable" || state.StandardUpdatesAvailable != 0 || state.ESMSecurityUpdatesAvailable != 3 {
@@ -37,6 +39,10 @@ func TestCollectAndRenderCompliance(t *testing.T) {
 		"gha_fleet_host_esm_security_updates_available 3\n",
 		`gha_fleet_host_kernel_info{release="6.8.0-138-generic"} 1`,
 		`gha_fleet_host_srso_status{status="vulnerable"} 1`,
+		// The drifted counter and the attributed truth ship side by side:
+		// the 2026-09-01 member showed ~1 GiB against 44 MiB.
+		"gha_fleet_host_slab_unreclaimable_counter_bytes 1091696640\n",
+		"gha_fleet_host_slab_unreclaimable_attributed_bytes 46323712\n",
 	} {
 		if !strings.Contains(metrics, wanted) {
 			t.Fatalf("metrics missing %q\n%s", wanted, metrics)
@@ -46,6 +52,10 @@ func TestCollectAndRenderCompliance(t *testing.T) {
 
 func TestComplianceFailsObservableWhenInputsAreMissing(t *testing.T) {
 	state := CollectCompliance(t.TempDir(), time.Now().UTC())
+	if state.SlabUnreclaimableCounterBytes != -1 || state.SlabUnreclaimableAttributedBytes != -1 {
+		t.Fatalf("absent slab inputs must read -1, got %v / %v",
+			state.SlabUnreclaimableCounterBytes, state.SlabUnreclaimableAttributedBytes)
+	}
 	if state.Complete || state.PackageInventoryAgeSeconds != -1 || state.SRSOStatus != "unknown" {
 		t.Fatalf("state=%#v", state)
 	}

internal/slabheal/slabheal.go

@@ -63,6 +63,39 @@ func Decide(facts Facts) Decision {
 	return Decision{Heal: true, Reason: fmt.Sprintf("SUnreclaim %d exceeds the %d budget on a quiet open member", facts.SUnreclaimBytes, facts.ThresholdBytes)}
 }
 
+// PreferAttributed picks the measurement the guard should act on. The
+// global meminfo counter drifts under container churn (measured 1041 MiB
+// against a 44 MiB memcg-attributed truth on 2026-09-01, the gap
+// unattributable to any live cache), so when the root cgroup exposes
+// slab_unreclaimable that attributed value wins; the counter remains the
+// fallback for hosts without cgroup v2 memory accounting.
+func PreferAttributed(attributed uint64, attributedOK bool, counter uint64) (uint64, string) {
+	if attributedOK {
+		return attributed, "memcg-attributed"
+	}
+	return counter, "meminfo-counter"
+}
+
+// ParseAttributedSlabUnreclaimable reads slab_unreclaimable (bytes) from
+// cgroup v2 memory.stat content.
+func ParseAttributedSlabUnreclaimable(reader io.Reader) (uint64, error) {
+	scanner := bufio.NewScanner(reader)
+	for scanner.Scan() {
+		fields := strings.Fields(scanner.Text())
+		if len(fields) == 2 && fields[0] == "slab_unreclaimable" {
+			value, err := strconv.ParseUint(fields[1], 10, 64)
+			if err != nil {
+				return 0, fmt.Errorf("malformed slab_unreclaimable value %q", fields[1])
+			}
+			return value, nil
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return 0, err
+	}
+	return 0, fmt.Errorf("memory.stat has no slab_unreclaimable line")
+}
+
 // ParseSUnreclaim reads SUnreclaim from /proc/meminfo content.
 func ParseSUnreclaim(reader io.Reader) (uint64, error) {
 	scanner := bufio.NewScanner(reader)

internal/slabheal/slabheal_test.go

@@ -52,3 +52,33 @@ func TestParseSUnreclaim(t *testing.T) {
 		t.Fatal("missing SUnreclaim line was accepted")
 	}
 }
+
+func TestParseAttributedSlabUnreclaimable(t *testing.T) {
+	t.Parallel()
+	value, err := ParseAttributedSlabUnreclaimable(strings.NewReader(
+		"anon 135168\nslab_reclaimable 766643200\nslab_unreclaimable 46323712\nslab 812966912\n",
+	))
+	if err != nil || value != 46323712 {
+		t.Fatalf("value=%d err=%v", value, err)
+	}
+	if _, err := ParseAttributedSlabUnreclaimable(strings.NewReader("anon 1\n")); err == nil {
+		t.Fatal("missing slab_unreclaimable line must error")
+	}
+	if _, err := ParseAttributedSlabUnreclaimable(strings.NewReader("slab_unreclaimable x\n")); err == nil {
+		t.Fatal("malformed value must error")
+	}
+}
+
+// The guard must act on the memcg-attributed truth when the kernel exposes
+// it: the global counter overstated a member by ~1 GiB on 2026-09-01.
+func TestPreferAttributedPicksTruthOverTheDriftingCounter(t *testing.T) {
+	t.Parallel()
+	value, source := PreferAttributed(46323712, true, 1091696640)
+	if value != 46323712 || source != "memcg-attributed" {
+		t.Fatalf("value=%d source=%s", value, source)
+	}
+	value, source = PreferAttributed(0, false, 1091696640)
+	if value != 1091696640 || source != "meminfo-counter" {
+		t.Fatalf("fallback value=%d source=%s", value, source)
+	}
+}