diff --git a/lib/diskutilization/README.md b/lib/diskutilization/README.md new file mode 100644 index 00000000..ba642376 --- /dev/null +++ b/lib/diskutilization/README.md @@ -0,0 +1,56 @@ +# Disk Utilization + +This package measures the actual on-disk footprint that Hypeman is consuming so operators can answer a different question than the existing allocation metrics. + +`hypeman_resources_disk_breakdown_bytes` remains the allocation and provisioned view. +`hypeman_disk_utilization_bytes` is the actual filesystem utilization view. + +The utilization metric reports bytes allocated on disk for these components: + +- `images` +- `oci_cache` +- `volumes` +- `rootfs_overlays` +- `volume_overlays` +- `snapshot_uncompressed` +- `snapshot_compressed` +- `snapshot_other` + +## How Measurement Works + +The measurement is done by walking only the known Hypeman storage roots instead of scanning the whole data filesystem. + +- `images` is measured from exported image disk files such as `rootfs.erofs` or `rootfs.ext4` +- `oci_cache` is measured from the OCI cache tree +- `volumes` is measured from volume `data.raw` files +- `rootfs_overlays` is measured from each guest `overlay.raw` +- `volume_overlays` is measured from each guest `vol-overlays/*.raw` +- snapshots are measured from each guest `snapshots/snapshot-latest` directory + +The measurement is based on filesystem allocated blocks rather than logical file size. That means sparse disks and overlays report the bytes they really occupy on disk, not the size they were provisioned with. + +Concretely, the collector reads filesystem metadata and uses the allocated block count for each file or directory entry, so the result reflects actual blocks consumed on disk. Directory walks are limited to Hypeman-managed paths that are already known from the data layout. + +Snapshots are classified by the memory artifact present in `snapshot-latest`: + +- `snapshot_compressed` for compressed memory files such as `memory-ranges.zst` or `memory-ranges.lz4` +- `snapshot_uncompressed` for raw `memory-ranges` +- `snapshot_other` when a snapshot directory exists but does not match a recognized memory artifact shape + +The full `snapshot-latest` directory is counted once under its classified snapshot component so the metric includes related config and state files, not just the memory artifact itself. + +## How It Is Stored For Metrics + +This package returns a per-component breakdown to the resource monitoring refresh loop. The refresh loop stores the latest measured values in the in-memory monitoring snapshot, and the Prometheus callback only reads that cached snapshot. + +That means: + +- the expensive filesystem work happens on the refresh interval +- the `/metrics` scrape path does not walk the filesystem +- each scrape simply emits the latest cached component values + +## Efficiency Expectations + +This should be efficient enough for the current design because it avoids whole-filesystem scans and avoids any disk walking during Prometheus scrapes. + +The main cost is the periodic walk over Hypeman-managed storage roots. That is still proportional to the number of tracked files and snapshot directories, so it is not free, but it is bounded and predictable. For v1, this is a good tradeoff: correct sparse-file accounting, accurate snapshot classification, cheap scrapes, and much lower complexity than a change-tracking cache. diff --git a/lib/diskutilization/diskutilization.go b/lib/diskutilization/diskutilization.go new file mode 100644 index 00000000..6cc46a66 --- /dev/null +++ b/lib/diskutilization/diskutilization.go @@ -0,0 +1,229 @@ +package diskutilization + +import ( + "io/fs" + "os" + "path/filepath" + "syscall" + + "github.com/kernel/hypeman/lib/paths" +) + +const ( + ComponentImages = "images" + ComponentOCICache = "oci_cache" + ComponentVolumes = "volumes" + ComponentRootfsOverlays = "rootfs_overlays" + ComponentVolumeOverlays = "volume_overlays" + ComponentSnapshotUncompressed = "snapshot_uncompressed" + ComponentSnapshotCompressed = "snapshot_compressed" + ComponentSnapshotOther = "snapshot_other" +) + +type Breakdown struct { + Images int64 + OCICache int64 + Volumes int64 + RootfsOverlays int64 + VolumeOverlays int64 + SnapshotUncompressed int64 + SnapshotCompressed int64 + SnapshotOther int64 +} + +func (b Breakdown) Components() map[string]int64 { + return map[string]int64{ + ComponentImages: b.Images, + ComponentOCICache: b.OCICache, + ComponentVolumes: b.Volumes, + ComponentRootfsOverlays: b.RootfsOverlays, + ComponentVolumeOverlays: b.VolumeOverlays, + ComponentSnapshotUncompressed: b.SnapshotUncompressed, + ComponentSnapshotCompressed: b.SnapshotCompressed, + ComponentSnapshotOther: b.SnapshotOther, + } +} + +func Collect(p *paths.Paths) (Breakdown, error) { + var breakdown Breakdown + + var err error + breakdown.Images, err = sumMatchingFilesAllocatedBytes(p.ImagesDir(), func(path string, entry fs.DirEntry) bool { + if entry.IsDir() { + return false + } + name := entry.Name() + return name == "rootfs.erofs" || name == "rootfs.ext4" + }) + if err != nil { + return Breakdown{}, err + } + + breakdown.OCICache, err = sumTreeAllocatedBytes(p.SystemOCICache()) + if err != nil { + return Breakdown{}, err + } + + breakdown.Volumes, err = sumDirectChildFileAllocatedBytes(p.VolumesDir(), "data.raw") + if err != nil { + return Breakdown{}, err + } + + guestEntries, err := os.ReadDir(p.GuestsDir()) + if err != nil { + if os.IsNotExist(err) { + return breakdown, nil + } + return Breakdown{}, err + } + + for _, guest := range guestEntries { + if !guest.IsDir() { + continue + } + + instanceID := guest.Name() + + breakdown.RootfsOverlays += allocatedBytesForPath(p.InstanceOverlay(instanceID)) + + volumeOverlays, err := sumMatchingFilesAllocatedBytes(p.InstanceVolumeOverlaysDir(instanceID), func(path string, entry fs.DirEntry) bool { + return !entry.IsDir() && filepath.Ext(entry.Name()) == ".raw" + }) + if err != nil { + return Breakdown{}, err + } + breakdown.VolumeOverlays += volumeOverlays + + snapshotDir := p.InstanceSnapshotLatest(instanceID) + classification, exists, err := classifySnapshotDir(snapshotDir) + if err != nil { + return Breakdown{}, err + } + if !exists { + continue + } + + snapshotBytes, err := sumTreeAllocatedBytes(snapshotDir) + if err != nil { + return Breakdown{}, err + } + + switch classification { + case ComponentSnapshotCompressed: + breakdown.SnapshotCompressed += snapshotBytes + case ComponentSnapshotUncompressed: + breakdown.SnapshotUncompressed += snapshotBytes + default: + breakdown.SnapshotOther += snapshotBytes + } + } + + return breakdown, nil +} + +func classifySnapshotDir(snapshotDir string) (component string, exists bool, err error) { + info, err := os.Stat(snapshotDir) + if err != nil { + if os.IsNotExist(err) { + return "", false, nil + } + return "", false, err + } + if !info.IsDir() { + return ComponentSnapshotOther, true, nil + } + + switch { + case pathExists(filepath.Join(snapshotDir, "memory-ranges.zst")): + return ComponentSnapshotCompressed, true, nil + case pathExists(filepath.Join(snapshotDir, "memory-ranges.lz4")): + return ComponentSnapshotCompressed, true, nil + case pathExists(filepath.Join(snapshotDir, "memory-ranges")): + return ComponentSnapshotUncompressed, true, nil + default: + return ComponentSnapshotOther, true, nil + } +} + +func sumDirectChildFileAllocatedBytes(root string, childFile string) (int64, error) { + entries, err := os.ReadDir(root) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + + var total int64 + for _, entry := range entries { + if !entry.IsDir() { + continue + } + total += allocatedBytesForPath(filepath.Join(root, entry.Name(), childFile)) + } + + return total, nil +} + +func sumMatchingFilesAllocatedBytes(root string, match func(path string, entry fs.DirEntry) bool) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if match(path, entry) { + total += allocatedBytesForPath(path) + } + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + return total, nil +} + +func sumTreeAllocatedBytes(root string) (int64, error) { + var total int64 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + total += allocatedBytesForPath(path) + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return 0, nil + } + return 0, err + } + return total, nil +} + +func allocatedBytesForPath(path string) int64 { + info, err := os.Lstat(path) + if err != nil { + return 0 + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0 + } + + return stat.Blocks * 512 +} + +func pathExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/lib/diskutilization/diskutilization_test.go b/lib/diskutilization/diskutilization_test.go new file mode 100644 index 00000000..cf22bed3 --- /dev/null +++ b/lib/diskutilization/diskutilization_test.go @@ -0,0 +1,115 @@ +package diskutilization + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" +) + +type sparseWrite struct { + offset int64 + data []byte +} + +func TestCollect_UsesAllocatedBytesAndClassifiesSnapshots(t *testing.T) { + p := paths.New(t.TempDir()) + + imagePath := filepath.Join(p.ImagesDir(), "library", "sha256:abc", "rootfs.erofs") + require.NoError(t, createSparseTestFile(imagePath, 32*1024*1024, []sparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("i"), 4096)}, + })) + + ociBlobPath := filepath.Join(p.SystemOCICache(), "blobs", "sha256", "blob") + require.NoError(t, createSparseTestFile(ociBlobPath, 16*1024, []sparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("o"), 8192)}, + })) + + volumePath := p.VolumeData("vol-1") + require.NoError(t, createSparseTestFile(volumePath, 64*1024*1024, []sparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("v"), 4096)}, + })) + + rootfsOverlayPath := p.InstanceOverlay("inst-1") + require.NoError(t, createSparseTestFile(rootfsOverlayPath, 64*1024*1024, []sparseWrite{ + {offset: 4 * 1024 * 1024, data: bytes.Repeat([]byte("r"), 4096)}, + })) + + volumeOverlayPath := p.InstanceVolumeOverlay("inst-1", "vol-1") + require.NoError(t, createSparseTestFile(volumeOverlayPath, 64*1024*1024, []sparseWrite{ + {offset: 8 * 1024 * 1024, data: bytes.Repeat([]byte("x"), 4096)}, + })) + + uncompressedSnapshotDir := p.InstanceSnapshotLatest("inst-1") + require.NoError(t, createSparseTestFile(filepath.Join(uncompressedSnapshotDir, "memory-ranges"), 64*1024*1024, []sparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("m"), 4096)}, + })) + require.NoError(t, os.WriteFile(filepath.Join(uncompressedSnapshotDir, "config.json"), []byte(`{}`), 0644)) + + compressedSnapshotDir := p.InstanceSnapshotLatest("inst-2") + require.NoError(t, createSparseTestFile(filepath.Join(compressedSnapshotDir, "memory-ranges.zst"), 32*1024*1024, []sparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("c"), 4096)}, + })) + require.NoError(t, os.WriteFile(filepath.Join(compressedSnapshotDir, "config.json"), []byte(`{}`), 0644)) + + otherSnapshotDir := p.InstanceSnapshotLatest("inst-3") + require.NoError(t, os.MkdirAll(otherSnapshotDir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(otherSnapshotDir, "config.json"), []byte(`{}`), 0644)) + + utilization, err := Collect(p) + require.NoError(t, err) + + require.Equal(t, allocatedBytesForPath(imagePath), utilization.Images) + require.Equal(t, allocatedBytesForPath(volumePath), utilization.Volumes) + require.Equal(t, allocatedBytesForPath(rootfsOverlayPath), utilization.RootfsOverlays) + require.Equal(t, allocatedBytesForPath(volumeOverlayPath), utilization.VolumeOverlays) + + require.Less(t, utilization.Volumes, fileSize(t, volumePath)) + require.Less(t, utilization.RootfsOverlays, fileSize(t, rootfsOverlayPath)) + + uncompressedTotal, err := sumTreeAllocatedBytes(uncompressedSnapshotDir) + require.NoError(t, err) + compressedTotal, err := sumTreeAllocatedBytes(compressedSnapshotDir) + require.NoError(t, err) + otherTotal, err := sumTreeAllocatedBytes(otherSnapshotDir) + require.NoError(t, err) + + require.Equal(t, uncompressedTotal, utilization.SnapshotUncompressed) + require.Equal(t, compressedTotal, utilization.SnapshotCompressed) + require.Equal(t, otherTotal, utilization.SnapshotOther) +} + +func createSparseTestFile(path string, size int64, writes []sparseWrite) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + if err := f.Truncate(size); err != nil { + return err + } + + for _, write := range writes { + if _, err := f.WriteAt(write.data, write.offset); err != nil { + return err + } + } + + return f.Sync() +} + +func fileSize(t *testing.T, path string) int64 { + t.Helper() + + info, err := os.Stat(path) + require.NoError(t, err) + return info.Size() +} diff --git a/lib/otel/README.md b/lib/otel/README.md index 47cd9123..6f1fa944 100644 --- a/lib/otel/README.md +++ b/lib/otel/README.md @@ -72,6 +72,19 @@ This keeps pull and push views aligned because both are sourced from the same OT | `hypeman_network_allocations_total` | gauge | | Active IP allocations | | `hypeman_network_tap_operations_total` | counter | operation | TAP create/delete ops | +### Resources +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `hypeman_resources_capacity` | gauge | resource | Raw host capacity by resource type | +| `hypeman_resources_effective_limit` | gauge | resource | Effective allocatable limit after oversubscription | +| `hypeman_resources_allocated` | gauge | resource | Current allocation by resource type | +| `hypeman_resources_oversub_ratio` | gauge | resource | Oversubscription ratio by resource type | +| `hypeman_resources_disk_breakdown_bytes` | gauge | component | Allocation/provisioned disk breakdown by component | +| `hypeman_disk_utilization_bytes` | gauge | component | Actual filesystem bytes allocated by disk component | +| `hypeman_resources_image_storage_bytes` | gauge | kind | Current and maximum image storage bytes | +| `hypeman_resources_gpu_slots` | gauge | kind | Total and used GPU slots | +| `hypeman_resources_gpu_profile_slots` | gauge | profile, kind | Available GPU slots by profile | + ### Volumes | Metric | Type | Description | |--------|------|-------------| diff --git a/lib/resources/monitoring.go b/lib/resources/monitoring.go index 322a532a..fa614a39 100644 --- a/lib/resources/monitoring.go +++ b/lib/resources/monitoring.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/kernel/hypeman/lib/diskutilization" "github.com/kernel/hypeman/lib/logger" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" @@ -24,6 +25,7 @@ type monitoringSnapshot struct { status FullResourceStatus imageStorageCurrent int64 imageStorageMax int64 + diskUtilization diskutilization.Breakdown } func (m *Manager) StartMonitoring(ctx context.Context, meter metric.Meter, refreshInterval time.Duration) error { @@ -106,6 +108,12 @@ func (m *Manager) refreshMonitoringSnapshot(ctx context.Context) error { } snapshot.imageStorageMax = m.MaxImageStorageBytes() + diskUtilization, err := diskutilization.Collect(m.paths) + if err != nil { + return err + } + snapshot.diskUtilization = diskUtilization + m.monitoring.mu.Lock() m.monitoring.snapshot = snapshot m.monitoring.hasSnapshot = true @@ -167,6 +175,15 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { return err } + diskUtilization, err := meter.Int64ObservableGauge( + "hypeman_disk_utilization_bytes", + metric.WithDescription("Actual disk bytes allocated on the filesystem by component"), + metric.WithUnit("By"), + ) + if err != nil { + return err + } + imageStorage, err := meter.Int64ObservableGauge( "hypeman_resources_image_storage_bytes", metric.WithDescription("Current and maximum image storage bytes"), @@ -220,6 +237,10 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { o.ObserveInt64(diskBreakdown, snapshot.status.DiskDetail.Overlays, metric.WithAttributes(attribute.String("component", "overlays"))) o.ObserveInt64(imageStorage, snapshot.imageStorageCurrent, metric.WithAttributes(attribute.String("kind", "current"))) } + + for component, value := range snapshot.diskUtilization.Components() { + o.ObserveInt64(diskUtilization, value, metric.WithAttributes(attribute.String("component", component))) + } o.ObserveInt64(imageStorage, snapshot.imageStorageMax, metric.WithAttributes(attribute.String("kind", "max"))) if snapshot.status.GPU != nil { @@ -236,7 +257,7 @@ func newMonitoringMetrics(meter metric.Meter, mgr *Manager) error { } return nil - }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, imageStorage, gpuSlots, gpuProfileSlots); err != nil { + }, capacity, effectiveLimit, allocated, oversubRatio, diskBreakdown, diskUtilization, imageStorage, gpuSlots, gpuProfileSlots); err != nil { return err } diff --git a/lib/resources/monitoring_test.go b/lib/resources/monitoring_test.go index d19751a6..2520d8e0 100644 --- a/lib/resources/monitoring_test.go +++ b/lib/resources/monitoring_test.go @@ -1,13 +1,18 @@ package resources import ( + "bytes" "context" + "os" + "path/filepath" "sync" + "syscall" "testing" "time" "github.com/kernel/hypeman/cmd/api/config" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/diskutilization" "github.com/kernel/hypeman/lib/paths" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel/attribute" @@ -38,6 +43,11 @@ type monitoringImageLister struct { ociCacheBytes int64 } +type monitoringSparseWrite struct { + offset int64 + data []byte +} + func (m *monitoringImageLister) TotalImageBytes(ctx context.Context) (int64, error) { m.mu.RLock() defer m.mu.RUnlock() @@ -131,6 +141,8 @@ func TestStartMonitoringPublishesCapacityMetrics(t *testing.T) { require.Equal(t, status.DiskDetail.OCICache, int64GaugeValue(t, rm, "hypeman_resources_disk_breakdown_bytes", map[string]string{"component": "oci_cache"})) require.Equal(t, status.DiskDetail.Volumes, int64GaugeValue(t, rm, "hypeman_resources_disk_breakdown_bytes", map[string]string{"component": "volumes"})) require.Equal(t, status.DiskDetail.Overlays, int64GaugeValue(t, rm, "hypeman_resources_disk_breakdown_bytes", map[string]string{"component": "overlays"})) + require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_disk_utilization_bytes", map[string]string{"component": "images"})) + require.Equal(t, int64(0), int64GaugeValue(t, rm, "hypeman_disk_utilization_bytes", map[string]string{"component": "snapshot_other"})) currentImageStorage := status.DiskDetail.Images + status.DiskDetail.OCICache require.Equal(t, currentImageStorage, int64GaugeValue(t, rm, "hypeman_resources_image_storage_bytes", map[string]string{"kind": "current"})) @@ -206,6 +218,92 @@ func TestStartMonitoringPublishesGPUMetrics(t *testing.T) { require.Equal(t, int64(2), int64GaugeValue(t, rm, "hypeman_resources_gpu_profile_slots", map[string]string{"profile": "L40S-2Q", "kind": "available"})) } +func TestStartMonitoringPublishesDiskUtilizationFromCachedSnapshot(t *testing.T) { + mgr, _, _ := monitoringTestManager(t) + + volumePath := mgr.paths.VolumeData("vol-1") + require.NoError(t, createMonitoringSparseTestFile(volumePath, 64*1024*1024, []monitoringSparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("v"), 4096)}, + })) + rootfsOverlayPath := mgr.paths.InstanceOverlay("vm-1") + require.NoError(t, createMonitoringSparseTestFile(rootfsOverlayPath, 64*1024*1024, []monitoringSparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("o"), 4096)}, + })) + snapshotDir := mgr.paths.InstanceSnapshotLatest("vm-1") + require.NoError(t, createMonitoringSparseTestFile(filepath.Join(snapshotDir, "memory-ranges.zst"), 32*1024*1024, []monitoringSparseWrite{ + {offset: 0, data: bytes.Repeat([]byte("s"), 4096)}, + })) + require.NoError(t, os.WriteFile(filepath.Join(snapshotDir, "config.json"), []byte(`{}`), 0644)) + + reader := otelmetric.NewManualReader() + provider := otelmetric.NewMeterProvider(otelmetric.WithReader(reader)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + require.NoError(t, mgr.StartMonitoring(ctx, provider.Meter("test"), time.Hour)) + + initialRM := collectMonitoringMetrics(t, reader) + initialVolumeBytes := int64GaugeValue(t, initialRM, "hypeman_disk_utilization_bytes", map[string]string{"component": "volumes"}) + initialCompressedSnapshotBytes := int64GaugeValue(t, initialRM, "hypeman_disk_utilization_bytes", map[string]string{"component": diskutilization.ComponentSnapshotCompressed}) + require.Equal(t, allocatedBytesForMonitoringPath(volumePath), initialVolumeBytes) + require.Equal(t, int64(100*1024*1024*1024), int64GaugeValue(t, initialRM, "hypeman_resources_disk_breakdown_bytes", map[string]string{"component": "volumes"})) + require.Greater(t, initialCompressedSnapshotBytes, int64(0)) + + f, err := os.OpenFile(volumePath, os.O_WRONLY, 0) + require.NoError(t, err) + _, err = f.WriteAt(bytes.Repeat([]byte("m"), 4096), 8*1024*1024) + require.NoError(t, err) + require.NoError(t, f.Close()) + + cachedRM := collectMonitoringMetrics(t, reader) + require.Equal(t, initialVolumeBytes, int64GaugeValue(t, cachedRM, "hypeman_disk_utilization_bytes", map[string]string{"component": "volumes"})) + + require.NoError(t, mgr.refreshMonitoringSnapshot(context.Background())) + + refreshedRM := collectMonitoringMetrics(t, reader) + refreshedVolumeBytes := int64GaugeValue(t, refreshedRM, "hypeman_disk_utilization_bytes", map[string]string{"component": "volumes"}) + require.Greater(t, refreshedVolumeBytes, initialVolumeBytes) +} + +func createMonitoringSparseTestFile(path string, size int64, writes []monitoringSparseWrite) error { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + if err := f.Truncate(size); err != nil { + return err + } + + for _, write := range writes { + if _, err := f.WriteAt(write.data, write.offset); err != nil { + return err + } + } + + return f.Sync() +} + +func allocatedBytesForMonitoringPath(path string) int64 { + info, err := os.Lstat(path) + if err != nil { + return 0 + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0 + } + + return stat.Blocks * 512 +} + func collectMonitoringMetrics(t *testing.T, reader *otelmetric.ManualReader) metricdata.ResourceMetrics { t.Helper()