From ff20311cf813a6e38278255f13bdeae2c5d91d68 Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Fri, 13 Nov 2020 01:01:27 -0800 Subject: [PATCH] added tar.gz downloading, hashing zzz for path --- .gitignore | 3 + build.zig | 36 ++-- libs/hzzp | 2 +- libs/zig-network | 2 +- libs/zzz | 2 +- src/commands.zig | 71 ++++--- src/import.zig | 478 +++++++++++++++++++++++++++++++++++------------ src/manifest.zig | 97 ++++++++++ src/tar.zig | 122 ++++++++++++ 9 files changed, 638 insertions(+), 175 deletions(-) create mode 100644 src/manifest.zig create mode 100644 src/tar.zig diff --git a/.gitignore b/.gitignore index 2040c29d..3fec57df 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,4 @@ zig-cache +zig-deps +deps.zig +imports.zzz diff --git a/build.zig b/build.zig index f28075d4..745abb92 100644 --- a/build.zig +++ b/build.zig @@ -1,30 +1,29 @@ usingnamespace std.build; const std = @import("std"); +const ssl = @import(pkgs.ssl.path); -const ssl = @import("libs/zig-bearssl/bearssl.zig"); - -const pkgs = [_]Pkg{ - Pkg{ +const pkgs = .{ + .clap = .{ .name = "clap", .path = "libs/zig-clap/clap.zig", }, - Pkg{ + .http = .{ .name = "http", .path = "libs/hzzp/src/main.zig", }, - Pkg{ + .net = .{ .name = "net", .path = "libs/zig-network/network.zig", }, - Pkg{ + .ssl = .{ .name = "ssl", .path = "libs/zig-bearssl/bearssl.zig", }, - Pkg{ + .uri = .{ .name = "uri", .path = "libs/zuri/src/zuri.zig", }, - Pkg{ + .zzz = .{ .name = "zzz", .path = "libs/zzz/src/main.zig", }, @@ -42,23 +41,26 @@ pub fn build(b: *Builder) void { exe.setTarget(target); exe.setBuildMode(mode); - for (pkgs) |pkg| { - exe.addPackage(pkg); + const tests = b.addTest("src/import.zig"); + tests.setBuildMode(mode); + inline for (std.meta.fields(@TypeOf(pkgs))) |field| { + exe.addPackage(@field(pkgs, field.name)); + tests.addPackage(@field(pkgs, field.name)); } ssl.linkBearSSL("libs/zig-bearssl", exe, target); - exe.linkSystemLibrary("git2"); - exe.linkSystemLibrary("openssl"); - exe.linkSystemLibrary("crypto"); - exe.linkSystemLibrary("ssh2"); - exe.linkSystemLibrary("zlib"); - exe.linkSystemLibrary("pcre"); exe.linkLibC(); exe.install(); const run_cmd = exe.run(); run_cmd.step.dependOn(b.getInstallStep()); + if (b.args) |args| { + run_cmd.addArgs(args); + } const run_step = b.step("run", "Run zkg"); run_step.dependOn(&run_cmd.step); + + const test_step = b.step("test", "Run tests"); + run_step.dependOn(&tests.step); } diff --git a/libs/hzzp b/libs/hzzp index f3d243aa..bc04e31f 160000 --- a/libs/hzzp +++ b/libs/hzzp @@ -1 +1 @@ -Subproject commit f3d243aa3e113dc49d9c24a8992d9b8605f89fb9 +Subproject commit bc04e31f5e2f4e6525afc82e192bd27e8e9b7967 diff --git a/libs/zig-network b/libs/zig-network index def6423d..ca711e54 160000 --- a/libs/zig-network +++ b/libs/zig-network @@ -1 +1 @@ -Subproject commit def6423dd23f07837a02c29b11cacae606e37b29 +Subproject commit ca711e54bde80fc15f8641f37c111dc4a3a7c221 diff --git a/libs/zzz b/libs/zzz index efd035c7..1b26d032 160000 --- a/libs/zzz +++ b/libs/zzz @@ -1 +1 @@ -Subproject commit efd035c7f160e830fec01ded0f76526deacd7097 +Subproject commit 1b26d03229b8ac87c851a69aa444872040a22327 diff --git a/src/commands.zig b/src/commands.zig index b8d7fe63..9eae4b44 100644 --- a/src/commands.zig +++ b/src/commands.zig @@ -4,8 +4,8 @@ const http = @import("http"); const net = @import("net"); const ssl = @import("ssl"); const Uri = @import("uri").Uri; -const CertificateValidator = @import("certificate_validator.zig"); const Manifest = @import("manifest.zig"); +const Import = @import("import.zig").Import; const os = std.os; const debug = std.debug; @@ -17,9 +17,6 @@ const fmt = std.fmt; const Allocator = mem.Allocator; const ArrayList = std.ArrayList; const OutStream = std.fs.File.OutStream; -const ChildProcess = std.ChildProcess; -const ExecResult = std.ChildProcess.ExecResult; -const Builder = std.build.Builder; const stdout = std.io.getStdOut().writer(); const stderr = std.io.getStdErr().writer(); @@ -34,7 +31,7 @@ const DependencyGraph = struct { cache: []const u8, queue_start: usize, nodes: std.ArrayList(Node), - results: ArrayList(ExecResult), + manifests: ArrayList(Manifest), const Self = @This(); @@ -45,7 +42,7 @@ const DependencyGraph = struct { .cache = cache, .queue_start = 0, .nodes = ArrayList(Node).init(allocator), - .results = ArrayList(ExecResult).init(allocator), + .manifests = ArrayList(Manifest).init(allocator), }; // TODO: get cwd of process @@ -56,9 +53,8 @@ const DependencyGraph = struct { fn deinit(self: *Self) void { self.nodes.deinit(); - for (self.results.span()) |result| { - self.allocator.free(result.stdout); - self.allocator.free(result.stderr); + for (self.manifests.items) |manifest| { + manifest.deinit(); } } @@ -69,31 +65,31 @@ const DependencyGraph = struct { const import_path = try std.fs.path.join(self.allocator, &[_][]const u8{ front.base_path, imports_zzz }); defer self.allocator.free(import_path); - std.debug.print("import path: {}, node: {}\n", .{ import_path, front }); + std.debug.print("import path: {}\n", .{import_path}); - const file = try std.fs.cwd().openFile(import_path, .{ .read = true, .write = true }); - defer file.close(); - - var manifest = try Manifest.init(self.allocator, file); - defer manifest.deinit(self.allocator); - - for (manifest.deps.items) |dep| { - const deps_dir = try std.fs.path.join(self.allocator, &[_][]const u8{ self.cache, "deps" }); - defer self.allocator.free(deps_dir); + const file = std.fs.cwd().openFile(import_path, .{ .read = true }) catch |err| { + if (err == error.FileNotFound) + continue + else + return err; + }; + try self.manifests.append(try Manifest.init(self.allocator, file)); - try dep.fetch(self.allocator, deps_dir); - const path = try dep.path(self.allocator, deps_dir); + var manifest = &self.manifests.items[self.manifests.items.len - 1]; + for (manifest.*.deps.items) |dep| { + const path = try dep.path(self.allocator, self.cache); defer self.allocator.free(path); - std.debug.print("path: {}\n", .{path}); + std.debug.print("name: {}, path: {}\n", .{ dep.name, path }); for (self.nodes.items) |*node| { if (mem.eql(u8, path, node.base_path)) { - try front.connect_dependency(node, dep.name, dep.root orelse "exports.zig"); + try front.connect_dependency(node, dep.name, dep.root); break; } } else { + try dep.fetch(self.allocator, self.cache); try self.nodes.append(try Node.init(self.allocator, path, front.depth + 1)); - try front.connect_dependency(&self.nodes.items[self.nodes.items.len - 1], dep.name, dep.root orelse "exports.zig"); + try front.connect_dependency(&self.nodes.items[self.nodes.items.len - 1], dep.name, dep.root); } try self.validate(); @@ -226,17 +222,13 @@ pub fn fetch(cache_path: ?[]const u8) !void { return err; }; - const cache = cache_path orelse "zig-cache"; - var dep_graph = try DependencyGraph.init(allocator, ".", cache); + var dep_graph = try DependencyGraph.init(allocator, ".", "zig-deps"); try dep_graph.process(); - var cache_dir = fs.Dir{ .fd = try os.open(cache, os.O_DIRECTORY, 0) }; - defer cache_dir.close(); - - const gen_file = try cache_dir.createFile("packages.zig", fs.File.CreateFlags{ + const gen_file = try std.fs.cwd().createFile("deps.zig", fs.File.CreateFlags{ .truncate = true, }); - errdefer cache_dir.deleteFile("packages.zig") catch {}; + errdefer std.fs.cwd().deleteFile("deps.zig") catch {}; defer gen_file.close(); const file_stream = gen_file.outStream(); @@ -301,7 +293,13 @@ fn https_request( var trust_anchor = ssl.TrustAnchorCollection.init(allocator); defer trust_anchor.deinit(); - try trust_anchor.appendFromPEM(ziglibs_pem); + const certs = try std.fs.openFileAbsolute("/etc/ssl/cert.pem", .{ .read = true }); + defer certs.close(); + + const pem = try certs.readToEndAlloc(allocator, 0x100000); + defer allocator.free(pem); + + try trust_anchor.appendFromPEM(pem); var x509 = ssl.x509.Minimal.init(trust_anchor); var ssl_client = ssl.Client.init(x509.getEngine()); ssl_client.relocate(); @@ -625,7 +623,7 @@ pub fn add( defer file.close(); var manifest = try Manifest.init(allocator, file); - defer manifest.deinit(allocator); + defer manifest.deinit(); const response = try query(allocator, remote_opt orelse default_remote, .{ .packages = .{ .name = name }, @@ -651,11 +649,8 @@ pub fn add( const entry = try Entry.from_json(root.Array.items[0]); try manifest.addImport(.{ .name = alias, - // ziglibs gives us git for now - .type = .git, - .src = entry.git, - .version = "master", .root = entry.root_file, + .src = try Import.urlToSource(entry.git), }); try manifest.commit(); @@ -666,7 +661,7 @@ pub fn remove(allocator: *Allocator, name: []const u8) !void { defer file.close(); var manifest = try Manifest.init(allocator, file); - defer manifest.deinit(allocator); + defer manifest.deinit(); try manifest.removeImport(name); try manifest.commit(); diff --git a/src/import.zig b/src/import.zig index 7eace0e8..1730868c 100644 --- a/src/import.zig +++ b/src/import.zig @@ -1,151 +1,395 @@ -usingnamespace std.os; const std = @import("std"); -const mem = std.mem; -const debug = std.debug; -const c = @cImport({ - @cInclude("git2.h"); -}); +const net = @import("net"); +const ssl = @import("ssl"); +const http = @import("http"); +const Uri = @import("uri").Uri; +const tar = @import("tar.zig"); +const zzz = @import("zzz"); -const default_root = "exports.zig"; const Allocator = std.mem.Allocator; +const gzipStream = std.compress.gzip.gzipStream; -name: []const u8, -type: Type, -src: []const u8, -version: ?[]const u8, -root: ?[]const u8, +pub const Import = struct { + name: []const u8, + root: []const u8, + src: Source, + integrity: ?Integrity = null, -const Self = @This(); + const Self = @This(); + const Hasher = std.crypto.hash.blake2.Blake2b128; -pub const Type = enum { - git, + const Source = union(enum) { + github: Github, + url: []const u8, - pub fn toString(self: Type) []const u8 { - inline for (std.meta.fields(Type)) |field| { - if (@field(Type, field.name) == self) { - return field.name; + const Github = struct { + user: []const u8, + repo: []const u8, + ref: []const u8, + }; + + fn addToZNode(source: Source, root: *zzz.ZNode, tree: anytype) !void { + if (@typeInfo(@TypeOf(tree)) != .Pointer) { + @compileError("tree must be pointer"); + } + + switch (source) { + .github => |github| { + var node = try tree.addNode(root, .{ .String = "github" }); + _ = try tree.addNode(node, .{ .String = github.user }); + _ = try tree.addNode(node, .{ .String = github.repo }); + _ = try tree.addNode(node, .{ .String = github.ref }); + }, + .url => |url| { + var node = try tree.addNode(root, .{ .String = "url" }); + _ = try tree.addNode(node, .{ .String = url }); + }, } } + + fn fromZNode(node: *const zzz.ZNode) !Source { + const key = try getZNodeString(node); + return if (std.mem.eql(u8, "github", key)) blk: { + var repo: ?[]const u8 = null; + var user: ?[]const u8 = null; + var ref: ?[]const u8 = null; + + var child = node.*.child; + while (child) |elem| : (child = child.?.sibling) { + const gh_key = try getZNodeString(elem); + + if (std.mem.eql(u8, "repo", gh_key)) { + repo = try getZNodeString(elem.child orelse return error.MissingRepo); + } else if (std.mem.eql(u8, "user", gh_key)) { + user = try getZNodeString(elem.child orelse return error.MissingUser); + } else if (std.mem.eql(u8, "ref", gh_key)) { + ref = try getZNodeString(elem.child orelse return error.MissingRef); + } else { + std.debug.print("unknown gh_key: {}\n", .{gh_key}); + return error.UnknownKey; + } + } + + break :blk Source{ + .github = .{ + .repo = repo orelse return error.MissingRepo, + .user = user orelse return error.MissingUser, + .ref = ref orelse return error.MissingRef, + }, + }; + } else if (std.mem.eql(u8, "url", key)) + Source{ .url = try getZNodeString(node.*.child orelse return error.MissingUrl) } + else { + std.debug.print("unknown key: {}\n", .{key}); + return error.UnknownKey; + }; + } + }; + + const Integrity = union(enum) { + sha256: []const u8, + + fn fromZNode(node: *const zzz.ZNode) !Integrity { + const key = try getZNodeString(node); + + const hash_type = try getZNodeString(node.*.child orelse return error.MissingHash); + const digest = try getZNodeString(node.*.child orelse return error.MissingDigest); + + return if (std.mem.eql(u8, "sha256", hash_type)) + Integrity{ .sha256 = digest } + else + error.UnknownHashType; + } + }; + + fn getZNodeString(node: *const zzz.ZNode) ![]const u8 { + return switch (node.value) { + .String => |str| str, + else => return error.NotAString, + }; } - pub fn fromString(str: []const u8) !Type { - return inline for (std.meta.fields(Type)) |field| { - if (std.mem.eql(u8, str, field.name)) { - break @field(Type, field.name); + pub fn fromZNode(node: *const zzz.ZNode) !Import { + const name = switch (node.value) { + .String => |str| str, + else => return error.MissingName, + }; + + var root_path: ?[]const u8 = null; + var src: ?Source = null; + var integrity: ?Integrity = null; + + var child = node.*.child; + while (child) |elem| : (child = child.?.sibling) { + const key = try getZNodeString(elem); + + if (std.mem.eql(u8, "root", key)) { + root_path = try getZNodeString(elem); + } else if (std.mem.eql(u8, "src", key)) { + src = try Source.fromZNode(elem.child orelse return error.MissingSourceType); + } else if (std.mem.eql(u8, "integrity", key)) { + integrity = try Integrity.fromZNode(elem.child orelse return error.MissingHashType); + } else { + std.debug.print("unknown key: {}\n", .{key}); + return error.UnknownKey; } - } else error.InvalidTypeName; + } + + return Import{ + .name = name, + .root = root_path orelse "src/main.zig", + .src = src orelse return error.MissingSource, + .integrity = integrity, + }; } -}; -pub fn fetch(self: Self, allocator: *Allocator, deps_dir: []const u8) !void { - return switch (self.type) { - .git => self.gitFetch(allocator, deps_dir), - }; -} + pub fn addToZNode(self: Self, root: *zzz.ZNode, tree: anytype) !void { + if (@typeInfo(@TypeOf(tree)) != .Pointer) { + @compileError("tree must be pointer"); + } -pub fn path(self: Self, allocator: *Allocator, deps_dir: []const u8) ![]const u8 { - return switch (self.type) { - .git => self.gitPath(allocator, deps_dir), - }; -} - -pub const GitError = error{ - Ok, - Error, - NotFound, - Exists, - Ambiguous, - Buf, - User, - BareRepo, - UnbornBranch, - Unmerged, - NonFastForward, - InvalidSpec, - Conflict, - Locked, - Modified, - Auth, - Certificate, - Applied, - Peel, - Eof, - Invalid, - Uncommitted, - Directory, - MergeConflict, - Passthrough, - IteratorOver, - Retry, - Mismatch, - DirtyIndex, - ApplyFail, + const import = try tree.addNode(root, .{ .String = self.name }); + const root_path = try tree.addNode(import, .{ .String = "root" }); + _ = try tree.addNode(root_path, .{ .String = self.root }); + + const src = try tree.addNode(import, .{ .String = "src" }); + try self.src.addToZNode(src, tree); + + if (self.integrity) |integrity| { + const integ_node = try tree.addNode(import, .{ .String = "integrity" }); + switch (integrity) { + .sha256 => |sha256| _ = try tree.addNode(integ_node, .{ .String = sha256 }), + } + } + } + + pub fn urlToSource(url: []const u8) !Source { + return Source{ .url = "" }; + } + + pub fn toUrl(self: Import, allocator: *Allocator) ![]const u8 { + return switch (self.src) { + .github => |github| try std.mem.join(allocator, "/", &[_][]const u8{ + "https://api.github.com/repos", + github.user, + github.repo, + "tarball", + github.ref, + }), + .url => |url| try allocator.dupe(u8, url), + }; + } + + pub fn path(self: Self, allocator: *Allocator, base_path: []const u8) ![]const u8 { + const digest = try self.hash(); + return std.fs.path.join(allocator, &[_][]const u8{ base_path, &digest }); + } + + pub fn hash(self: Self) ![Hasher.digest_length * 2]u8 { + var tree = zzz.ZTree(1, 100){}; + var root = try tree.addNode(null, .Null); + try self.src.addToZNode(root, &tree); + + var buf: [std.mem.page_size]u8 = undefined; + var digest: [Hasher.digest_length]u8 = undefined; + var ret: [Hasher.digest_length * 2]u8 = undefined; + var fixed_buffer = std.io.fixedBufferStream(&buf); + + try root.stringify(fixed_buffer.writer()); + Hasher.hash(fixed_buffer.getWritten(), &digest, .{}); + + const lookup = [_]u8{ '0', '1', '2', '3', '4', '5', '6', '7', '8', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + for (digest) |val, i| { + ret[2 * i] = lookup[val >> 4]; + ret[(2 * i) + 1] = lookup[@truncate(u4, val)]; + } + + return ret; + } + + pub fn fetch(self: Self, allocator: *Allocator, deps_path: []const u8) !void { + var source = try HttpsSource.init(allocator, self); + defer source.deinit(); + + // TODO: integrity check here + + var gzip = try gzipStream(allocator, source.reader()); + defer gzip.deinit(); + + var deps_dir = try std.fs.cwd().makeOpenPath(deps_path, .{ .access_sub_paths = true }); + defer deps_dir.close(); + + const digest = try self.hash(); + var dest_dir = try deps_dir.makeOpenPath(&digest, .{ .access_sub_paths = true }); + defer dest_dir.close(); + + try tar.instantiate(allocator, dest_dir, gzip.reader(), 1); + } }; -fn fetchSubmodule(submodule: ?*c.git_submodule, name: [*c]const u8, payload: ?*c_void) callconv(.C) c_int { - const repo = @ptrCast(*c.git_repository, payload); - var status = c.git_submodule_set_fetch_recurse_submodules(repo, name, @intToEnum(c.git_submodule_recurse_t, 1)); - if (status == -1) return status; +const Connection = struct { + ssl_client: ssl.Client, + ssl_socket: SslStream, + socket: net.Socket, + socket_reader: net.Socket.Reader, + socket_writer: net.Socket.Writer, + http_buf: [std.mem.page_size]u8, + http_client: HttpClient, + window: []const u8, + + const SslStream = ssl.Stream(*net.Socket.Reader, *net.Socket.Writer); + const HttpClient = http.base.Client.Client(SslStream.DstInStream, SslStream.DstOutStream); + const Self = @This(); + + pub fn init(allocator: *Allocator, hostname: [:0]const u8, port: u16, x509: *ssl.x509.Minimal) !*Self { + var ret = try allocator.create(Self); + errdefer allocator.destroy(ret); - var opts: c.git_submodule_update_options = undefined; - status = c.git_submodule_update_options_init(&opts, c.GIT_SUBMODULE_UPDATE_OPTIONS_VERSION); - if (status == -1) return status; + ret.window = &[_]u8{}; + ret.ssl_client = ssl.Client.init(x509.getEngine()); + ret.ssl_client.relocate(); + try ret.ssl_client.reset(hostname, false); - return c.git_submodule_update(submodule, 1, &opts); -} + ret.socket = try net.connectToHost(allocator, hostname, port, .tcp); + errdefer ret.socket.close(); -fn gitFetch(self: Self, allocator: *Allocator, deps_dir: []const u8) !void { - var repo: ?*c.git_repository = undefined; - var opts: c.git_clone_options = undefined; + ret.socket_reader = ret.socket.reader(); + ret.socket_writer = ret.socket.writer(); - var status = c.git_clone_options_init(&opts, c.GIT_CLONE_OPTIONS_VERSION); - if (status == -1) { - return error.GitCloneOptionsInit; + ret.ssl_socket = ssl.initStream( + ret.ssl_client.getEngine(), + &ret.socket_reader, + &ret.socket_writer, + ); + errdefer ret.ssl_socket.close catch {}; + + ret.http_client = http.base.Client.create( + &ret.http_buf, + ret.ssl_socket.inStream(), + ret.ssl_socket.outStream(), + ); + + return ret; } - opts.checkout_branch = self.version.?.ptr; + pub fn deinit(self: *Self) void { + self.ssl_socket.close() catch {}; + self.socket.close(); + } - const location = try self.path(allocator, deps_dir); - defer allocator.free(location); + const ReadError = HttpClient.ReadError || error{AbruptClose}; + pub const Reader = std.io.Reader(*Connection, ReadError, read); - const url = try std.cstr.addNullByte(allocator, self.src); - defer allocator.free(url); + fn copyToBuf(self: *Self, buffer: []u8) usize { + const len = std.math.min(buffer.len, self.window.len); + std.mem.copy(u8, buffer[0..len], self.window[0..len]); + self.window = self.window[len..]; + return len; + } - debug.print("location: {}\n", .{location}); - status = c.git_clone(&repo, url, location.ptr, &opts); - if (status < 0 and status != -4) { - //const err = @ptrCast(*const c.git_error, c.git_error_last()); - //debug.print("clone issue: ({}) {}\n", .{ status, @ptrCast([*:0]const u8, err.message) }); - return error.GitClone; + fn read(self: *Self, buffer: []u8) ReadError!usize { + return if (self.window.len != 0) + self.copyToBuf(buffer) + else if (try self.http_client.readEvent()) |event| blk: { + switch (event) { + .closed => { + std.debug.print("got close\n", .{}); + break :blk 0; + }, + .chunk => |chunk| { + self.window = chunk.data; + break :blk self.copyToBuf(buffer); + }, + else => |val| { + std.debug.print("something else: {}\n", .{val}); + break :blk @as(usize, 0); + }, + } + } else 0; } - // recursively checkout submodules - status = c.git_submodule_foreach(repo, fetchSubmodule, repo); - if (status == -1) { - return error.RecursiveSubmoduleCheckout; + pub fn reader(self: *Self) Reader { + return .{ .context = self }; } -} +}; + +const HttpsSource = struct { + allocator: *Allocator, + trust_anchor: ssl.TrustAnchorCollection, + x509: ssl.x509.Minimal, + connection: *Connection, + + const Self = @This(); + + pub fn init(allocator: *Allocator, import: Import) !Self { + var url = try import.toUrl(allocator); + defer allocator.free(url); + + const file = try std.fs.openFileAbsolute("/etc/ssl/cert.pem", .{ .read = true }); + defer file.close(); + + const certs = try file.readToEndAlloc(allocator, 500000); + defer allocator.free(certs); + + var trust_anchor = ssl.TrustAnchorCollection.init(allocator); + try trust_anchor.appendFromPEM(certs); + + var x509 = ssl.x509.Minimal.init(trust_anchor); -fn gitPath(self: Self, allocator: *Allocator, deps_dir: []const u8) ![]const u8 { - return try std.cstr.addNullByte( - allocator, - try std.mem.join(allocator, std.fs.path.sep_str, &[_][]const u8{ - deps_dir, - try gitUrlToName(self.src), - self.version.?, - }), - ); -} + var conn: *Connection = undefined; + redirect: while (true) { + const uri = try Uri.parse(url, true); + const port = uri.port orelse 443; -fn gitUrlToName(url: []const u8) ![]const u8 { - const https = "https://"; - const dot_git = ".git"; + if (!std.mem.eql(u8, uri.scheme, "https")) return error.HttpsOnly; - // TODO: validate ssh url - if (mem.startsWith(u8, url, https)) { - const end = if (mem.endsWith(u8, url, dot_git)) url.len - dot_git.len else url.len; - return url[https.len..end]; + const hostname = try std.cstr.addNullByte(allocator, uri.host.name); + defer allocator.free(hostname); + + conn = try Connection.init(allocator, hostname, port, &x509); + try conn.http_client.writeHead("GET", uri.path); + try conn.http_client.writeHeaderValue("Host", hostname); + try conn.http_client.writeHeaderValue("User-Agent", "zkg"); + try conn.http_client.writeHeaderValue("Accept", "*/*"); + try conn.http_client.writeHeadComplete(); + try conn.ssl_socket.flush(); + + var redirect = false; + while (try conn.http_client.readEvent()) |event| { + switch (event) { + .status => |status| switch (status.code) { + 200 => {}, + 302 => redirect = true, + else => return error.HttpFailed, + }, + .header => |header| { + if (redirect and std.mem.eql(u8, "location", header.name)) { + allocator.free(url); + url = try allocator.dupe(u8, header.value); + conn.deinit(); + continue :redirect; + } + }, + .head_complete => break :redirect, + else => |val| std.debug.print("got other: {}\n", .{val}), + } + } + } + + return Self{ + .allocator = allocator, + .trust_anchor = trust_anchor, + .x509 = x509, + .connection = conn, + }; } - return error.UnsupportedUrl; -} + pub fn deinit(self: *Self) void { + self.connection.deinit(); + self.trust_anchor.deinit(); + } + + pub fn reader(self: *Self) Connection.Reader { + return self.connection.reader(); + } +}; diff --git a/src/manifest.zig b/src/manifest.zig new file mode 100644 index 00000000..0bc60944 --- /dev/null +++ b/src/manifest.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const zzz = @import("zzz"); +const Import = @import("import.zig").Import; +const Allocator = std.mem.Allocator; + +allocator: *Allocator, +file: std.fs.File, +deps: std.ArrayList(Import), +text: []const u8, + +const Self = @This(); +const ZTree = zzz.ZTree(1, 1000); + +const ChildIterator = struct { + val: ?*const zzz.ZNode, + + fn next(self: *ChildIterator) ?*const zzz.ZNode { + return if (self.val) |node| blk: { + self.val = node.sibling; + break :blk node; + } else null; + } + + fn init(node: *const zzz.ZNode) ChildIterator { + return ChildIterator{ + .val = node.child, + }; + } +}; + +/// if path doesn't exist, create it else load contents +pub fn init(allocator: *Allocator, file: std.fs.File) !Self { + var deps = std.ArrayList(Import).init(allocator); + errdefer deps.deinit(); + + const text = try file.readToEndAlloc(allocator, 0x2000); + errdefer allocator.free(text); + + var tree = ZTree{}; + var root = try tree.appendText(text); + + // iterate and append to deps + var import_it = ChildIterator.init(root); + while (import_it.next()) |node| { + try deps.append(try Import.fromZNode(node)); + } + + return Self{ + .allocator = allocator, + .file = file, + .deps = deps, + .text = text, + }; +} + +/// on destruction serialize to file +pub fn deinit(self: *Self) void { + self.file.close(); + self.deps.deinit(); + self.allocator.free(self.text); +} + +pub fn commit(self: *Self) !void { + var tree = ZTree{}; + var root = try tree.addNode(null, .Null); + + for (self.deps.items) |dep| { + _ = try dep.addToZNode(root, &tree); + } + + _ = try self.file.seekTo(0); + _ = try self.file.setEndPos(0); + try root.stringifyPretty(self.file.writer()); +} + +pub fn addImport(self: *Self, import: Import) !void { + if (import.name.len == 0) { + return error.EmptyName; + } + + for (self.deps.items) |dep| { + if (std.mem.eql(u8, dep.name, import.name)) { + return error.NameExists; + } + } + + try self.deps.append(import); +} + +pub fn removeImport(self: *Self, name: []const u8) !void { + for (self.deps.items) |dep, i| { + if (std.mem.eql(u8, dep.name, name)) { + _ = self.deps.orderedRemove(i); + break; + } + } else return error.NameNotFound; +} diff --git a/src/tar.zig b/src/tar.zig new file mode 100644 index 00000000..001d34d8 --- /dev/null +++ b/src/tar.zig @@ -0,0 +1,122 @@ +const std = @import("std"); + +const testing = std.testing; +const Allocator = std.mem.Allocator; + +// ustar tar implementation +pub const Header = extern struct { + name: [100]u8, + mode: [8]u8, + uid: [8]u8, + gid: [8]u8, + size: [11:0]u8, + mtime: [12]u8, + checksum: [8]u8, + typeflag: FileType, + linkname: [100]u8, + magic: [6]u8, + version: [2]u8, + uname: [32]u8, + gname: [32]u8, + devmajor: [8]u8, + devminor: [8]u8, + prefix: [155]u8, + pad: [12]u8 = [_]u8{0} ** 12, + + const Self = @This(); + + const FileType = extern enum(u8) { + regular = '0', + hard_link = '1', + symbolic_link = '2', + character = '3', + block = '4', + directory = '5', + fifo = '6', + reserved = '7', + pax_global = 'g', + extended = 'x', + _, + }; + + pub fn isBlank(self: *const Header) bool { + const block = std.mem.asBytes(self); + return for (block) |elem| { + if (elem != 0) break false; + } else true; + } +}; + +test "Header size" { + testing.expectEqual(512, @sizeOf(Header)); +} + +pub fn instantiate(allocator: *Allocator, dir: std.fs.Dir, reader: anytype, skip_depth: usize) !void { + var count: usize = 0; + while (true) { + const header = reader.readStruct(Header) catch |err| { + return if (err == error.EndOfStream) if (count < 2) error.AbrubtEnd else break else err; + }; + + const block = std.mem.asBytes(&header); + if (header.isBlank()) { + count += 1; + continue; + } else if (count > 0) { + return error.Format; + } + + var size = try std.fmt.parseUnsigned(usize, &header.size, 8); + const block_size = ((size + 511) / 512) * 512; + var components = std.ArrayList([]const u8).init(allocator); + defer components.deinit(); + + var path_it = std.mem.tokenize(&header.prefix, "/\x00"); + if (header.prefix[0] != 0) { + while (path_it.next()) |component| { + try components.append(component); + } + } + + path_it = std.mem.tokenize(&header.name, "/\x00"); + while (path_it.next()) |component| { + try components.append(component); + } + + const tmp_path = try std.fs.path.join(allocator, components.items); + defer allocator.free(tmp_path); + + if (skip_depth >= components.items.len) { + try reader.skipBytes(block_size, .{}); + continue; + } + + var i: usize = 0; + while (i < skip_depth) : (i += 1) { + _ = components.orderedRemove(0); + } + + const file_path = try std.fs.path.join(allocator, components.items); + defer allocator.free(file_path); + + switch (header.typeflag) { + .directory => try dir.makePath(file_path), + .pax_global => try reader.skipBytes(512, .{}), + .regular => { + const file = try dir.createFile(file_path, .{ .read = true, .truncate = true }); + defer file.close(); + const skip_size = block_size - size; + + var buf: [std.mem.page_size]u8 = undefined; + while (size > 0) { + const buffered = try reader.read(buf[0..std.math.min(size, 512)]); + try file.writeAll(buf[0..buffered]); + size -= buffered; + } + + try reader.skipBytes(skip_size, .{}); + }, + else => {}, + } + } +}