Merge pull request ziglang#19388 from ziglang/cache-dedup

cache system file deduplication

lib/std/Build/Cache/Directory.zig

@@ -0,0 +1,74 @@
+const Directory = @This();
+const std = @import("../../std.zig");
+const fs = std.fs;
+const fmt = std.fmt;
+const Allocator = std.mem.Allocator;
+
+/// This field is redundant for operations that can act on the open directory handle
+/// directly, but it is needed when passing the directory to a child process.
+/// `null` means cwd.
+path: ?[]const u8,
+handle: fs.Dir,
+
+pub fn clone(d: Directory, arena: Allocator) Allocator.Error!Directory {
+    return .{
+        .path = if (d.path) |p| try arena.dupe(u8, p) else null,
+        .handle = d.handle,
+    };
+}
+
+pub fn cwd() Directory {
+    return .{
+        .path = null,
+        .handle = fs.cwd(),
+    };
+}
+
+pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
+    if (self.path) |p| {
+        // TODO clean way to do this with only 1 allocation
+        const part2 = try fs.path.join(allocator, paths);
+        defer allocator.free(part2);
+        return fs.path.join(allocator, &[_][]const u8{ p, part2 });
+    } else {
+        return fs.path.join(allocator, paths);
+    }
+}
+
+pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
+    if (self.path) |p| {
+        // TODO clean way to do this with only 1 allocation
+        const part2 = try fs.path.join(allocator, paths);
+        defer allocator.free(part2);
+        return fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
+    } else {
+        return fs.path.joinZ(allocator, paths);
+    }
+}
+
+/// Whether or not the handle should be closed, or the path should be freed
+/// is determined by usage, however this function is provided for convenience
+/// if it happens to be what the caller needs.
+pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
+    self.handle.close();
+    if (self.path) |p| gpa.free(p);
+    self.* = undefined;
+}
+
+pub fn format(
+    self: Directory,
+    comptime fmt_string: []const u8,
+    options: fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    _ = options;
+    if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
+    if (self.path) |p| {
+        try writer.writeAll(p);
+        try writer.writeAll(fs.path.sep_str);
+    }
+}
+
+pub fn eql(self: Directory, other: Directory) bool {
+    return self.handle.fd == other.handle.fd;
+}

lib/std/Build/Cache/Path.zig

@@ -0,0 +1,154 @@
+root_dir: Cache.Directory,
+/// The path, relative to the root dir, that this `Path` represents.
+/// Empty string means the root_dir is the path.
+sub_path: []const u8 = "",
+
+pub fn clone(p: Path, arena: Allocator) Allocator.Error!Path {
+    return .{
+        .root_dir = try p.root_dir.clone(arena),
+        .sub_path = try arena.dupe(u8, p.sub_path),
+    };
+}
+
+pub fn cwd() Path {
+    return .{ .root_dir = Cache.Directory.cwd() };
+}
+
+pub fn join(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
+    if (sub_path.len == 0) return p;
+    const parts: []const []const u8 =
+        if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
+    return .{
+        .root_dir = p.root_dir,
+        .sub_path = try fs.path.join(arena, parts),
+    };
+}
+
+pub fn resolvePosix(p: Path, arena: Allocator, sub_path: []const u8) Allocator.Error!Path {
+    if (sub_path.len == 0) return p;
+    return .{
+        .root_dir = p.root_dir,
+        .sub_path = try fs.path.resolvePosix(arena, &.{ p.sub_path, sub_path }),
+    };
+}
+
+pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
+    const parts: []const []const u8 =
+        if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
+    return p.root_dir.join(allocator, parts);
+}
+
+pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![:0]u8 {
+    const parts: []const []const u8 =
+        if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
+    return p.root_dir.joinZ(allocator, parts);
+}
+
+pub fn openFile(
+    p: Path,
+    sub_path: []const u8,
+    flags: fs.File.OpenFlags,
+) !fs.File {
+    var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+    const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+        break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+            p.sub_path, sub_path,
+        }) catch return error.NameTooLong;
+    };
+    return p.root_dir.handle.openFile(joined_path, flags);
+}
+
+pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
+    var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+    const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+        break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+            p.sub_path, sub_path,
+        }) catch return error.NameTooLong;
+    };
+    return p.root_dir.handle.makeOpenPath(joined_path, opts);
+}
+
+pub fn statFile(p: Path, sub_path: []const u8) !fs.Dir.Stat {
+    var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+    const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+        break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+            p.sub_path, sub_path,
+        }) catch return error.NameTooLong;
+    };
+    return p.root_dir.handle.statFile(joined_path);
+}
+
+pub fn atomicFile(
+    p: Path,
+    sub_path: []const u8,
+    options: fs.Dir.AtomicFileOptions,
+    buf: *[fs.MAX_PATH_BYTES]u8,
+) !fs.AtomicFile {
+    const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+        break :p std.fmt.bufPrint(buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+            p.sub_path, sub_path,
+        }) catch return error.NameTooLong;
+    };
+    return p.root_dir.handle.atomicFile(joined_path, options);
+}
+
+pub fn access(p: Path, sub_path: []const u8, flags: fs.File.OpenFlags) !void {
+    var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+    const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+        break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+            p.sub_path, sub_path,
+        }) catch return error.NameTooLong;
+    };
+    return p.root_dir.handle.access(joined_path, flags);
+}
+
+pub fn makePath(p: Path, sub_path: []const u8) !void {
+    var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
+    const joined_path = if (p.sub_path.len == 0) sub_path else p: {
+        break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
+            p.sub_path, sub_path,
+        }) catch return error.NameTooLong;
+    };
+    return p.root_dir.handle.makePath(joined_path);
+}
+
+pub fn format(
+    self: Path,
+    comptime fmt_string: []const u8,
+    options: std.fmt.FormatOptions,
+    writer: anytype,
+) !void {
+    if (fmt_string.len == 1) {
+        // Quote-escape the string.
+        const stringEscape = std.zig.stringEscape;
+        const f = switch (fmt_string[0]) {
+            'q' => "",
+            '\'' => '\'',
+            else => @compileError("unsupported format string: " ++ fmt_string),
+        };
+        if (self.root_dir.path) |p| {
+            try stringEscape(p, f, options, writer);
+            if (self.sub_path.len > 0) try stringEscape(fs.path.sep_str, f, options, writer);
+        }
+        if (self.sub_path.len > 0) {
+            try stringEscape(self.sub_path, f, options, writer);
+        }
+        return;
+    }
+    if (fmt_string.len > 0)
+        std.fmt.invalidFmtError(fmt_string, self);
+    if (self.root_dir.path) |p| {
+        try writer.writeAll(p);
+        try writer.writeAll(fs.path.sep_str);
+    }
+    if (self.sub_path.len > 0) {
+        try writer.writeAll(self.sub_path);
+        try writer.writeAll(fs.path.sep_str);
+    }
+}
+
+const Path = @This();
+const std = @import("../../std.zig");
+const fs = std.fs;
+const Allocator = std.mem.Allocator;
+const Cache = std.Build.Cache;

lib/std/Build/Step.zig

@@ -544,7 +544,7 @@ pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool {
 
 fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror {
     const i = man.failed_file_index orelse return err;
-    const pp = man.files.items[i].prefixed_path orelse return err;
+    const pp = man.files.keys()[i].prefixed_path;
     const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
     return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });
 }