diff --git a/doc/build.zig.zon.md b/doc/build.zig.zon.md index 690905a400af..944ef1f7f646 100644 --- a/doc/build.zig.zon.md +++ b/doc/build.zig.zon.md @@ -109,6 +109,13 @@ Boolean. When this is set to `true`, a package is declared to be lazily fetched. This makes the dependency only get fetched if it is actually used. +#### `no_unpack` + +String. Optional. + +When set, disables unpacking the package and instead saves it as a single file +using the given string as its file path. + ### `paths` List. Required. diff --git a/lib/init/build.zig.zon b/lib/init/build.zig.zon index 1dbe7519e7eb..76bef4d2554f 100644 --- a/lib/init/build.zig.zon +++ b/lib/init/build.zig.zon @@ -69,6 +69,10 @@ // // fetched. This makes the dependency only get fetched if it is // // actually used. // .lazy = false, + // + // // When set, disables unpacking the package and instead saves it as a single file + // // using the given string as its file path. + // //.no_unpack = "an/example/package", //}, }, diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index efee6605ed23..607fc3adb420 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -33,6 +33,7 @@ location_tok: std.zig.Ast.TokenIndex, hash_tok: std.zig.Ast.TokenIndex, name_tok: std.zig.Ast.TokenIndex, lazy_status: LazyStatus, +no_unpack: ?[]const u8, parent_package_root: Cache.Path, parent_manifest_ast: ?*const std.zig.Ast, prog_node: std.Progress.Node, @@ -351,12 +352,12 @@ pub fn run(f: *Fetch) RunError!void { .path_or_url => |path_or_url| { if (fs.cwd().openDir(path_or_url, .{ .iterate = true })) |dir| { var resource: Resource = .{ .dir = dir }; - return f.runResource(path_or_url, &resource, null); + return f.runResource(path_or_url, &resource, null, f.no_unpack); } else |dir_err| { const file_err = if (dir_err == error.NotDir) e: { if (fs.cwd().openFile(path_or_url, .{})) |file| { var resource: Resource = .{ .file = file }; - return f.runResource(path_or_url, &resource, null); + return f.runResource(path_or_url, &resource, null, f.no_unpack); } else |err| break :e err; } else dir_err; @@ -368,7 +369,7 @@ pub fn run(f: *Fetch) RunError!void { }; var server_header_buffer: [header_buffer_size]u8 = undefined; var resource = try f.initResource(uri, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, f.no_unpack); } }, }; @@ -434,7 +435,7 @@ pub fn run(f: *Fetch) RunError!void { ); var server_header_buffer: [header_buffer_size]u8 = undefined; var resource = try f.initResource(uri, &server_header_buffer); - return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); + return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, f.no_unpack); } pub fn deinit(f: *Fetch) void { @@ -448,6 +449,7 @@ fn runResource( uri_path: []const u8, resource: *Resource, remote_hash: ?Package.Hash, + no_unpack: ?[]const u8, ) RunError!void { defer resource.deinit(); const arena = f.arena.allocator(); @@ -478,7 +480,7 @@ fn runResource( defer tmp_directory.handle.close(); // Fetch and unpack a resource into a temporary directory. - var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); + var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory, no_unpack); var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; @@ -746,6 +748,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .hash_tok = dep.hash_tok, .name_tok = dep.name_tok, .lazy_status = if (dep.lazy) .available else .eager, + .no_unpack = dep.no_unpack, .parent_package_root = f.package_root, .parent_manifest_ast = &f.manifest_ast, .prog_node = f.prog_node, @@ -1075,8 +1078,43 @@ fn unpackResource( resource: *Resource, uri_path: []const u8, tmp_directory: Cache.Directory, + no_unpack: ?[]const u8, ) RunError!UnpackResult { const eb = &f.error_bundle; + + if (no_unpack) |sub_path| { + if (std.fs.path.dirname(sub_path)) |sub_dir| { + tmp_directory.handle.makePath(sub_dir) catch |e| return f.fail( + f.location_tok, + try eb.printString( + "failed to create temporary path '{}{s}' with {s}", + .{ tmp_directory, sub_dir, @errorName(e) }, + ), + ); + } + var out_file = tmp_directory.handle.createFile( + sub_path, + .{}, + ) catch |err| return f.fail(f.location_tok, try eb.printString( + "failed to create temporary file: {s}", + .{@errorName(err)}, + )); + defer out_file.close(); + var buf: [4096]u8 = undefined; + while (true) { + const len = resource.reader().readAll(&buf) catch |err| return f.fail(f.location_tok, try eb.printString( + "read stream failed: {s}", + .{@errorName(err)}, + )); + if (len == 0) break; + out_file.writer().writeAll(buf[0..len]) catch |err| return f.fail(f.location_tok, try eb.printString( + "write temporary file failed: {s}", + .{@errorName(err)}, + )); + } + return .{}; + } + const file_type = switch (resource.*) { .file => FileType.fromPath(uri_path) orelse return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), @@ -2317,6 +2355,7 @@ const TestFetchBuilder = struct { .hash_tok = 0, .name_tok = 0, .lazy_status = .eager, + .no_unpack = null, .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, .parent_manifest_ast = null, .prog_node = std.Progress.Node.none, diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index c526854df2d0..63653329a1f5 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -22,6 +22,7 @@ pub const Dependency = struct { node: Ast.Node.Index, name_tok: Ast.TokenIndex, lazy: bool, + no_unpack: ?[]const u8, pub const Location = union(enum) { url: []const u8, @@ -141,6 +142,15 @@ pub fn copyErrorsIntoBundle( } } +pub fn getNoUnpackProblem(no_unpack: []const u8) ?[]const u8 { + if (std.mem.indexOfScalar(u8, no_unpack, '\\') != null) return "may not contain backslashes"; + var it = std.fs.path.ComponentIterator(.posix, u8).init(no_unpack) catch return "is not a valid path"; + while (it.next()) |entry| { + if (std.mem.eql(u8, entry.name, "..")) return "may not contain '..' components"; + } + return null; +} + const Parse = struct { gpa: Allocator, ast: Ast, @@ -287,6 +297,7 @@ const Parse = struct { .node = node, .name_tok = 0, .lazy = false, + .no_unpack = null, }; var has_location = false; @@ -335,6 +346,19 @@ const Parse = struct { error.ParseFailure => continue, else => |e| return e, }; + } else if (mem.eql(u8, field_name, "no_unpack")) { + if (dep.no_unpack != null) return fail(p, main_tokens[field_init], "dependency should specify only one 'no_unpack' field", .{}); + + dep.no_unpack = parseString(p, field_init) catch |err| switch (err) { + error.ParseFailure => continue, + else => |e| return e, + }; + if (getNoUnpackProblem(dep.no_unpack.?)) |problem| return fail( + p, + main_tokens[field_init], + "no_unpack value {s}", + .{problem}, + ); } else { // Ignore unknown fields so that we can add fields in future zig // versions without breaking older zig versions. diff --git a/src/main.zig b/src/main.zig index 7b1bc50bd614..c5f83891f884 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5219,6 +5219,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .hash_tok = 0, .name_tok = 0, .lazy_status = .eager, + .no_unpack = null, .parent_package_root = build_mod.root, .parent_manifest_ast = null, .prog_node = fetch_prog_node, @@ -7022,6 +7023,7 @@ const usage_fetch = \\ --save=[name] Add the fetched package to build.zig.zon as name \\ --save-exact Add the fetched package to build.zig.zon, storing the URL verbatim \\ --save-exact=[name] Add the fetched package to build.zig.zon as name, storing the URL verbatim + \\ --no-unpack[=path] Don't unpack the package, save to path if gven \\ ; @@ -7043,6 +7045,7 @@ fn cmdFetch( yes: ?[]const u8, exact: ?[]const u8, } = .no; + var no_unpack: ?union(enum) { url_basename, with_path: []const u8 } = null; { var i: usize = 0; @@ -7067,6 +7070,13 @@ fn cmdFetch( save = .{ .exact = null }; } else if (mem.startsWith(u8, arg, "--save-exact=")) { save = .{ .exact = arg["--save-exact=".len..] }; + } else if (mem.eql(u8, arg, "--no-unpack")) { + no_unpack = .url_basename; + } else if (mem.startsWith(u8, arg, "--no-unpack=")) { + const path = arg["--no-unpack=".len..]; + if (Package.Manifest.getNoUnpackProblem(path)) |problem| + fatal("--no-unpack path {s}", .{problem}); + no_unpack = .{ .with_path = path }; } else { fatal("unrecognized parameter: '{s}'", .{arg}); } @@ -7121,6 +7131,10 @@ fn cmdFetch( .hash_tok = 0, .name_tok = 0, .lazy_status = .eager, + .no_unpack = if (no_unpack) |n| switch (n) { + .url_basename => std.fs.path.basename(path_or_url), + .with_path => |path| path, + } else null, .parent_package_root = undefined, .parent_manifest_ast = null, .prog_node = root_prog_node, diff --git a/test/standalone/build.zig.zon b/test/standalone/build.zig.zon index db1c7125a774..bdda8ecca5fb 100644 --- a/test/standalone/build.zig.zon +++ b/test/standalone/build.zig.zon @@ -69,6 +69,9 @@ .@"extern" = .{ .path = "extern", }, + .fetch = .{ + .path = "fetch", + }, .dep_diamond = .{ .path = "dep_diamond", }, diff --git a/test/standalone/fetch/build.zig b/test/standalone/fetch/build.zig new file mode 100644 index 000000000000..df14770fdc72 --- /dev/null +++ b/test/standalone/fetch/build.zig @@ -0,0 +1,105 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) !void { + const test_step = b.step("test", "Test it"); + b.default_step = test_step; + + const codegen_exe = b.addExecutable(.{ + .name = "codegen", + .target = b.graph.host, + .root_source_file = b.path("codegen.zig"), + }); + + { + const run_codegen = b.addRunArtifact(codegen_exe); + const example_dir = run_codegen.addOutputDirectoryArg("example"); + run_codegen.addArg("N-V-__8AABkAAAD5UlwyjuMCRESo0AsKtkhnbeeaA8Ux-LTi"); + run_codegen.addArg("subpath/example_dep_file.txt"); + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "build", + "install", + "--build-file", + }); + run.addFileArg(example_dir.path(b, "build.zig")); + run.addArg("--prefix"); + const install_dir = run.addOutputDirectoryArg("install"); + const check_file = b.addCheckFile(install_dir.path(b, "example_dep_file.txt"), .{ + .expected_exact = "This is an example file.\n", + }); + test_step.dependOn(&check_file.step); + } + + { + const run_codegen = b.addRunArtifact(codegen_exe); + const example_dir = run_codegen.addOutputDirectoryArg("example"); + run_codegen.addArg("N-V-__8AABkAAAD5UlwyjuMCRESo0AsKtkhnbeeaA8Ux-LTi"); + run_codegen.addArg("../foo.txt"); + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "build", + "install", + "--build-file", + }); + run.addFileArg(example_dir.path(b, "build.zig")); + run.addCheck(.{ .expect_stderr_match = "error: no_unpack value may not contain '..' components" }); + test_step.dependOn(&run.step); + } + + { + const run_codegen = b.addRunArtifact(codegen_exe); + const example_dir = run_codegen.addOutputDirectoryArg("example"); + run_codegen.addArg("N-V-__8AABkAAAD5UlwyjuMCRESo0AsKtkhnbeeaA8Ux-LTi"); + run_codegen.addArg("foo\\\\bar.txt"); + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "build", + "install", + "--build-file", + }); + run.addFileArg(example_dir.path(b, "build.zig")); + run.addCheck(.{ .expect_stderr_match = "error: no_unpack value may not contain backslashes" }); + test_step.dependOn(&run.step); + } + + { + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "fetch", + "--no-unpack", + }); + run.addFileArg(b.path("example/example_dep_file.txt")); + run.expectStdOutEqual("N-V-__8AABkAAAAPaKynDrx2BXIAr0Nqq1cg7FOngHE8XcYU\n"); + test_step.dependOn(&run.step); + } + { + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "fetch", + "--no-unpack=subpath/example_dep_file.txt", + }); + run.addFileArg(b.path("example/example_dep_file.txt")); + run.expectStdOutEqual("N-V-__8AABkAAAD5UlwyjuMCRESo0AsKtkhnbeeaA8Ux-LTi\n"); + test_step.dependOn(&run.step); + } + { + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "fetch", + "--no-unpack=subpath/../example_dep_file.txt", + }); + run.addFileArg(b.path("example/example_dep_file.txt")); + run.addCheck(.{ .expect_stderr_match = "error: --no-unpack path may not contain '..' components" }); + test_step.dependOn(&run.step); + } + { + const run = b.addSystemCommand(&.{ + b.graph.zig_exe, + "fetch", + "--no-unpack=subpath\\example_dep_file.txt", + }); + run.addFileArg(b.path("example/example_dep_file.txt")); + run.addCheck(.{ .expect_stderr_match = "error: --no-unpack path may not contain backslashes" }); + test_step.dependOn(&run.step); + } +} diff --git a/test/standalone/fetch/codegen.zig b/test/standalone/fetch/codegen.zig new file mode 100644 index 000000000000..1379bd178a19 --- /dev/null +++ b/test/standalone/fetch/codegen.zig @@ -0,0 +1,52 @@ +const std = @import("std"); + +pub fn main() !void { + var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); + const arena = arena_instance.allocator(); + const args = try std.process.argsAlloc(arena); + std.debug.assert(args.len == 4); + const out_dir = args[1]; + const hash = args[2]; + const no_unpack = args[3]; + + if (!std.fs.path.isAbsolute(out_dir)) { + std.log.err("directory '{s}' must be absolute", .{out_dir}); + std.process.exit(0xff); + } + + var dir = try std.fs.openDirAbsolute(out_dir, .{}); + defer dir.close(); + + try writeFile(dir, "example_dep_file.txt", @embedFile("example/example_dep_file.txt")); + + const package_path_absolute = try arena.dupe(u8, out_dir); + for (package_path_absolute) |*c| { + c.* = if (c.* == '\\') '/' else c.*; + } + const template: Template = .{ + .package_path_absolute = package_path_absolute, + .hash = hash, + .no_unpack = no_unpack, + }; + try writeFile(dir, "build.zig", try template.process(arena, @embedFile("example/build.zig.template"))); + try writeFile(dir, "build.zig.zon", try template.process(arena, @embedFile("example/build.zig.zon.template"))); +} + +const Template = struct { + package_path_absolute: []const u8, + hash: []const u8, + no_unpack: []const u8, + pub fn process(self: Template, arena: std.mem.Allocator, template: []const u8) ![]const u8 { + const content1 = try std.mem.replaceOwned(u8, arena, template, "", self.package_path_absolute); + defer arena.free(content1); + const content2 = try std.mem.replaceOwned(u8, arena, content1, "", self.hash); + defer arena.free(content2); + return try std.mem.replaceOwned(u8, arena, content2, "", self.no_unpack); + } +}; + +fn writeFile(dir: std.fs.Dir, name: []const u8, content: []const u8) !void { + const file = try dir.createFile(name, .{}); + defer file.close(); + try file.writer().writeAll(content); +} diff --git a/test/standalone/fetch/example/build.zig.template b/test/standalone/fetch/example/build.zig.template new file mode 100644 index 000000000000..61095a057b12 --- /dev/null +++ b/test/standalone/fetch/example/build.zig.template @@ -0,0 +1,9 @@ +const std = @import("std"); + +pub fn build(b: *std.Build) void { + const dep = b.dependency("somedependency", .{}); + b.getInstallStep().dependOn(&b.addInstallFile( + dep.path(""), + "example_dep_file.txt", + ).step); +} diff --git a/test/standalone/fetch/example/build.zig.zon.template b/test/standalone/fetch/example/build.zig.zon.template new file mode 100644 index 000000000000..47ef0f72f260 --- /dev/null +++ b/test/standalone/fetch/example/build.zig.zon.template @@ -0,0 +1,17 @@ +.{ + .name = .fetchexample, + .version = "0.0.0", + .fingerprint = 0xf7fb52913f1f8822, + .dependencies = .{ + .somedependency = .{ + .url = "file:///example_dep_file.txt", + .hash = "", + .no_unpack = "", + }, + }, + .paths = .{ + "build.zig", + "build.zig.zon", + "example_dep_file.txt", + }, +} diff --git a/test/standalone/fetch/example/example_dep_file.txt b/test/standalone/fetch/example/example_dep_file.txt new file mode 100644 index 000000000000..e738d768073c --- /dev/null +++ b/test/standalone/fetch/example/example_dep_file.txt @@ -0,0 +1 @@ +This is an example file.