diff --git a/lib/diskutilization/diskutilization.go b/lib/diskutilization/diskutilization.go index 0ef66eaa..27fef3c8 100644 --- a/lib/diskutilization/diskutilization.go +++ b/lib/diskutilization/diskutilization.go @@ -17,6 +17,7 @@ const ( ComponentVolumeOverlays = "volume_overlays" ComponentSnapshotUncompressed = "snapshot_uncompressed" ComponentSnapshotCompressed = "snapshot_compressed" + ComponentSnapshotShared = "snapshot_shared" ComponentSnapshotOther = "snapshot_other" ) @@ -28,6 +29,7 @@ type Breakdown struct { VolumeOverlays int64 SnapshotUncompressed int64 SnapshotCompressed int64 + SnapshotShared int64 SnapshotOther int64 } @@ -40,6 +42,7 @@ func (b Breakdown) Components() map[string]int64 { ComponentVolumeOverlays: b.VolumeOverlays, ComponentSnapshotUncompressed: b.SnapshotUncompressed, ComponentSnapshotCompressed: b.SnapshotCompressed, + ComponentSnapshotShared: b.SnapshotShared, ComponentSnapshotOther: b.SnapshotOther, } } @@ -77,6 +80,7 @@ func Collect(p *paths.Paths) (Breakdown, error) { return Breakdown{}, err } + sharedSnapshotExtents := newSharedExtentTracker() for _, guest := range guestEntries { if !guest.IsDir() { continue @@ -103,10 +107,11 @@ func Collect(p *paths.Paths) (Breakdown, error) { continue } - snapshotBytes, err := sumTreeAllocatedBytes(snapshotDir) + snapshotBytes, sharedSnapshotBytes, err := sumSnapshotTreeAllocatedBytes(snapshotDir, sharedSnapshotExtents) if err != nil { return Breakdown{}, err } + breakdown.SnapshotShared += sharedSnapshotBytes switch classification { case ComponentSnapshotCompressed: @@ -215,6 +220,30 @@ func sumTreeAllocatedBytes(root string) (int64, error) { return total, nil } +func sumSnapshotTreeAllocatedBytes(root string, sharedExtents *sharedExtentTracker) (int64, int64, error) { + var privateTotal int64 + var sharedTotal int64 + err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + privateBytes, sharedBytes := snapshotAllocatedBytesForPath(path, sharedExtents) + privateTotal += privateBytes + sharedTotal += sharedBytes + return nil + }) + if err != nil { + if os.IsNotExist(err) { + return 0, 0, nil + } + return 0, 0, err + } + return privateTotal, sharedTotal, nil +} + func allocatedBytesForPath(path string) int64 { info, err := os.Lstat(path) if err != nil { diff --git a/lib/diskutilization/diskutilization_reflink_linux.go b/lib/diskutilization/diskutilization_reflink_linux.go new file mode 100644 index 00000000..e567342b --- /dev/null +++ b/lib/diskutilization/diskutilization_reflink_linux.go @@ -0,0 +1,174 @@ +//go:build linux + +package diskutilization + +import ( + "os" + "sort" + "syscall" + "unsafe" + + "golang.org/x/sys/unix" +) + +const ( + fsIOCFiemap = 0xC020660B + fiemapFlagSync = 0x1 + fiemapExtentLast = 0x1 + fiemapExtentShared = 0x2000 + fiemapMaxExtents = 256 + fiemapWholeFile = ^uint64(0) +) + +type fiemapHeader struct { + Start uint64 + Length uint64 + Flags uint32 + MappedExtents uint32 + ExtentCount uint32 + Reserved uint32 +} + +type fiemapExtent struct { + Logical uint64 + Physical uint64 + Length uint64 + Reserved64 [2]uint64 + Flags uint32 + Reserved [3]uint32 +} + +type fiemapRequest struct { + Header fiemapHeader + Extents [fiemapMaxExtents]fiemapExtent +} + +type sharedExtentTracker struct { + ranges []physicalRange +} + +type physicalRange struct { + start uint64 + end uint64 +} + +func newSharedExtentTracker() *sharedExtentTracker { + return &sharedExtentTracker{} +} + +func snapshotAllocatedBytesForPath(path string, sharedExtents *sharedExtentTracker) (int64, int64) { + info, err := os.Lstat(path) + if err != nil { + return 0, 0 + } + if !info.Mode().IsRegular() { + return allocatedBytesForPath(path), 0 + } + + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return 0, 0 + } + allocatedBytes := stat.Blocks * 512 + if allocatedBytes == 0 { + return 0, 0 + } + + privateBytes, sharedBytes, sawShared, err := fiemapAllocatedBytes(path, sharedExtents) + if err != nil || !sawShared { + return allocatedBytes, 0 + } + return privateBytes, sharedBytes +} + +func fiemapAllocatedBytes(path string, sharedExtents *sharedExtentTracker) (int64, int64, bool, error) { + f, err := os.Open(path) + if err != nil { + return 0, 0, false, err + } + defer f.Close() + + var privateBytes int64 + var sharedBytes int64 + var sawShared bool + start := uint64(0) + + for { + var req fiemapRequest + req.Header.Start = start + req.Header.Length = fiemapWholeFile + req.Header.Flags = fiemapFlagSync + req.Header.ExtentCount = fiemapMaxExtents + + if _, _, errno := unix.Syscall(unix.SYS_IOCTL, f.Fd(), uintptr(fsIOCFiemap), uintptr(unsafe.Pointer(&req))); errno != 0 { + return 0, 0, false, errno + } + if req.Header.MappedExtents == 0 { + break + } + + extents := req.Extents[:req.Header.MappedExtents] + for _, extent := range extents { + if extent.Length == 0 { + continue + } + if extent.Flags&fiemapExtentShared == 0 { + privateBytes += int64(extent.Length) + continue + } + sawShared = true + sharedBytes += sharedExtents.add(extent.Physical, extent.Length) + } + + last := extents[len(extents)-1] + if last.Flags&fiemapExtentLast != 0 { + break + } + next := last.Logical + last.Length + if next <= start { + break + } + start = next + } + + return privateBytes, sharedBytes, sawShared, nil +} + +func (t *sharedExtentTracker) add(start, length uint64) int64 { + if length == 0 { + return 0 + } + + end := start + length + added := length + for _, existing := range t.ranges { + if existing.end <= start || end <= existing.start { + continue + } + overlapStart := max(start, existing.start) + overlapEnd := min(end, existing.end) + added -= overlapEnd - overlapStart + } + + t.ranges = append(t.ranges, physicalRange{start: start, end: end}) + t.compact() + return int64(added) +} + +func (t *sharedExtentTracker) compact() { + sort.Slice(t.ranges, func(i, j int) bool { + return t.ranges[i].start < t.ranges[j].start + }) + + merged := t.ranges[:0] + for _, current := range t.ranges { + if len(merged) == 0 || merged[len(merged)-1].end < current.start { + merged = append(merged, current) + continue + } + if current.end > merged[len(merged)-1].end { + merged[len(merged)-1].end = current.end + } + } + t.ranges = merged +} diff --git a/lib/diskutilization/diskutilization_reflink_linux_test.go b/lib/diskutilization/diskutilization_reflink_linux_test.go new file mode 100644 index 00000000..92ac619a --- /dev/null +++ b/lib/diskutilization/diskutilization_reflink_linux_test.go @@ -0,0 +1,87 @@ +//go:build linux + +package diskutilization + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/kernel/hypeman/lib/paths" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestCollect_CountsReflinkSnapshotExtentsOnce(t *testing.T) { + p := paths.New(t.TempDir()) + + srcPath := filepath.Join(p.InstanceSnapshotLatest("inst-1"), "memory-ranges") + dstPath := filepath.Join(p.InstanceSnapshotLatest("inst-2"), "memory-ranges") + require.NoError(t, os.MkdirAll(filepath.Dir(srcPath), 0755)) + require.NoError(t, os.MkdirAll(filepath.Dir(dstPath), 0755)) + require.NoError(t, os.WriteFile(srcPath, bytes.Repeat([]byte("m"), 4*1024*1024), 0644)) + + cloned, err := cloneTestFile(srcPath, dstPath) + require.NoError(t, err) + if !cloned { + t.Skip("FICLONE is not supported by this filesystem") + } + + _, _, sawShared, err := fiemapAllocatedBytes(srcPath, newSharedExtentTracker()) + if err != nil { + t.Skipf("FIEMAP is not supported by this filesystem: %v", err) + } + if !sawShared { + t.Skip("FICLONE did not produce FIEMAP shared extents") + } + + srcAllocated := allocatedBytesForPath(srcPath) + dstAllocated := allocatedBytesForPath(dstPath) + require.Greater(t, srcAllocated, int64(0)) + require.Equal(t, srcAllocated, dstAllocated) + + utilization, err := Collect(p) + require.NoError(t, err) + + require.InDelta(t, srcAllocated, utilization.SnapshotShared, float64(64*1024)) + require.Less(t, utilization.SnapshotUncompressed, srcAllocated) + require.Less(t, utilization.SnapshotUncompressed+utilization.SnapshotShared, srcAllocated+dstAllocated) +} + +func TestSharedExtentTrackerAdd(t *testing.T) { + tracker := newSharedExtentTracker() + + require.Equal(t, int64(100), tracker.add(1000, 100)) + require.Equal(t, int64(0), tracker.add(1000, 100)) + require.Equal(t, int64(50), tracker.add(1050, 100)) + require.Equal(t, int64(100), tracker.add(1200, 100)) + require.Equal(t, int64(100), tracker.add(900, 200)) +} + +func cloneTestFile(srcPath, dstPath string) (bool, error) { + src, err := os.Open(srcPath) + if err != nil { + return false, err + } + defer src.Close() + + dst, err := os.OpenFile(dstPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return false, err + } + defer dst.Close() + + if err := unix.IoctlFileClone(int(dst.Fd()), int(src.Fd())); err != nil { + if errors.Is(err, unix.EINVAL) || + errors.Is(err, unix.ENOTSUP) || + errors.Is(err, unix.EOPNOTSUPP) || + errors.Is(err, unix.EXDEV) || + errors.Is(err, unix.ENOTTY) { + return false, nil + } + return false, err + } + return true, nil +} diff --git a/lib/diskutilization/diskutilization_reflink_other.go b/lib/diskutilization/diskutilization_reflink_other.go new file mode 100644 index 00000000..eae27d2a --- /dev/null +++ b/lib/diskutilization/diskutilization_reflink_other.go @@ -0,0 +1,13 @@ +//go:build !linux + +package diskutilization + +type sharedExtentTracker struct{} + +func newSharedExtentTracker() *sharedExtentTracker { + return &sharedExtentTracker{} +} + +func snapshotAllocatedBytesForPath(path string, _ *sharedExtentTracker) (int64, int64) { + return allocatedBytesForPath(path), 0 +}