From 1ccc879f40f3c3ae6ae036996908db9153346ed3 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 10:16:18 +0800 Subject: [PATCH 1/8] Rework WPT runner We have no crashing tests, remove safe mode. Allows better re-use of arenas, and if we do introduce a crash, it won't be easy to ignore. Could allow for re-using the environment across tests to further improve performance. Remove console now that we have a working console api. --- src/browser/browser.zig | 2 +- src/main_wpt.zig | 607 ++++++++++++++++++++++------------------ src/wpt/fileloader.zig | 64 ----- src/wpt/run.zig | 151 ---------- src/wpt/testcase.zig | 263 ----------------- 5 files changed, 337 insertions(+), 750 deletions(-) delete mode 100644 src/wpt/fileloader.zig delete mode 100644 src/wpt/run.zig delete mode 100644 src/wpt/testcase.zig diff --git a/src/browser/browser.zig b/src/browser/browser.zig index 8b87164c0..3d35b603b 100644 --- a/src/browser/browser.zig +++ b/src/browser/browser.zig @@ -247,7 +247,7 @@ pub const Page = struct { .url = URL.empty, .session = session, .renderer = FlatRenderer.init(arena), - .window_clicked_event_node = .{ .func = windowClicked }, + .window_clicked_event_node = .{ .func = windowClicked }, .state = .{ .arena = arena, .document = null, diff --git a/src/main_wpt.zig b/src/main_wpt.zig index 0c36be3bd..9a70974a6 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -18,30 +18,16 @@ const std = @import("std"); -const wpt = @import("wpt/run.zig"); -const Suite = @import("wpt/testcase.zig").Suite; +const Allocator = std.mem.Allocator; +const ArenaAllocator = std.heap.ArenaAllocator; + +const Env = @import("browser/env.zig").Env; const Platform = @import("runtime/js.zig").Platform; -const FileLoader = @import("wpt/fileloader.zig").FileLoader; - -const wpt_dir = "tests/wpt"; - -const usage = - \\usage: {s} [options] [test filter] - \\ Run the Web Test Platform. - \\ - \\ -h, --help Print this help message and exit. - \\ --json result is formatted in JSON. - \\ --safe each test is run in a separate process. - \\ --summary print a summary result. Incompatible w/ --json - \\ -; - -// Out list all the ouputs handled by WPT. -const Out = enum { - json, - summary, - text, -}; + +const parser = @import("browser/netsurf.zig"); +const polyfill = @import("browser/polyfill/polyfill.zig"); + +const WPT_DIR = "tests/wpt"; pub const std_options = std.Options{ // Set the log level to info @@ -51,327 +37,406 @@ pub const std_options = std.Options{ // TODO For now the WPT tests run is specific to WPT. // It manually load js framwork libs, and run the first script w/ js content in // the HTML page. -// Once lightpanda will have the html loader, it would be useful to refacto +// Once lightpanda will have the html loader, it would be useful to refactor // this test to use it. pub fn main() !void { - var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + var gpa: std.heap.DebugAllocator(.{}) = .init; defer _ = gpa.deinit(); - const alloc = gpa.allocator(); + const allocator = gpa.allocator(); - var args = try std.process.argsWithAllocator(alloc); - defer args.deinit(); + // An arena for the runner itself, lives for the duration of the the process + var ra = ArenaAllocator.init(allocator); + defer ra.deinit(); + const runner_arena = ra.allocator(); - // get the exec name. - const execname = args.next().?; + const cmd = try parseArgs(runner_arena); - var out: Out = .text; - var safe = false; + const platform = try Platform.init(); + defer platform.deinit(); - var filter = std.ArrayList([]const u8).init(alloc); - defer filter.deinit(); + // prepare libraries to load on each test case. + var loader = FileLoader.init(runner_arena, WPT_DIR); - while (args.next()) |arg| { - if (std.mem.eql(u8, "-h", arg) or std.mem.eql(u8, "--help", arg)) { - try std.io.getStdErr().writer().print(usage, .{execname}); - std.posix.exit(0); - } - if (std.mem.eql(u8, "--json", arg)) { - out = .json; - continue; - } - if (std.mem.eql(u8, "--safe", arg)) { - safe = true; - continue; - } - if (std.mem.eql(u8, "--summary", arg)) { - out = .summary; + var it = try TestIterator.init(runner_arena, WPT_DIR, cmd.filters); + defer it.deinit(); + + var writer = try Writer.init(runner_arena, cmd.format); + + // An arena for running each tests. Is reset after every test. + var test_arena = ArenaAllocator.init(allocator); + defer test_arena.deinit(); + + while (try it.next()) |test_file| { + defer _ = test_arena.reset(.{ .retain_capacity = {} }); + + var err_out: ?[]const u8 = null; + const result = run(test_arena.allocator(), test_file, &loader, &err_out) catch |err| blk: { + if (err_out == null) { + err_out = @errorName(err); + } + break :blk null; + }; + + if (result == null and err_out == null) { + // We somtimes pass a non-test to `run` (we don't know it's a non + // test, we need to open the contents of the test file to find out + // and that's in run). continue; } - try filter.append(arg[0..]); - } - // summary is available in safe mode only. - if (out == .summary) { - safe = true; + try writer.process(test_file, result, err_out); } + try writer.finalize(); +} - // browse the dir to get the tests dynamically. - var list = std.ArrayList([]const u8).init(alloc); - try wpt.find(alloc, wpt_dir, &list); - defer { - for (list.items) |tc| { - alloc.free(tc); - } - list.deinit(); - } +fn run(arena: Allocator, test_file: []const u8, loader: *FileLoader, err_out: *?[]const u8) !?[]const u8 { + // document + const html = blk: { + const full_path = try std.fs.path.join(arena, &.{ WPT_DIR, test_file }); + const file = try std.fs.cwd().openFile(full_path, .{}); + defer file.close(); + break :blk try file.readToEndAlloc(arena, 128 * 1024); + }; - if (safe) { - return try runSafe(alloc, execname, out, list.items, filter.items); + if (std.mem.indexOf(u8, html, "testharness.js") == null) { + // This isn't a test. A lot of files are helpers/content for tests to + // make use of. + return null; } - var results = std.ArrayList(Suite).init(alloc); - defer { - for (results.items) |suite| { - suite.deinit(); - } - results.deinit(); - } + const dirname = std.fs.path.dirname(test_file) orelse unreachable; - // initialize VM JS lib. - const platform = try Platform.init(); - defer platform.deinit(); + var runner = try @import("testing.zig").jsRunner(arena, .{ + .html = html, + }); + defer runner.deinit(); - // prepare libraries to load on each test case. - var loader = FileLoader.init(alloc, wpt_dir); - defer loader.deinit(); + try polyfill.load(arena, runner.scope); - var run: usize = 0; - var failures: usize = 0; - for (list.items) |tc| { - if (!shouldRun(filter.items, tc)) { - continue; + // loop over the scripts. + const doc = parser.documentHTMLToDocument(runner.state.document.?); + const scripts = try parser.documentGetElementsByTagName(doc, "script"); + const script_count = try parser.nodeListLength(scripts); + for (0..script_count) |i| { + const s = (try parser.nodeListItem(scripts, @intCast(i))).?; + + // If the script contains an src attribute, load it. + if (try parser.elementGetAttribute(@as(*parser.Element, @ptrCast(s)), "src")) |src| { + var path = src; + if (!std.mem.startsWith(u8, src, "/")) { + path = try std.fs.path.join(arena, &.{ "/", dirname, path }); + } + const script_source = loader.get(path) catch |err| { + err_out.* = std.fmt.allocPrint(arena, "{s} - {s}", .{ @errorName(err), path }) catch null; + return err; + }; + try runner.exec(script_source, src, err_out); } - run += 1; + // If the script as a source text, execute it. + const src = try parser.nodeTextContent(s) orelse continue; + try runner.exec(src, null, err_out); + } - // create an arena and deinit it for each test case. - var arena = std.heap.ArenaAllocator.init(alloc); - defer arena.deinit(); + { + // Mark tests as ready to run. + const loadevt = try parser.eventCreate(); + defer parser.eventDestroy(loadevt); - var msg_out: ?[]const u8 = null; - const res = wpt.run(arena.allocator(), wpt_dir, tc, &loader, &msg_out) catch |err| { - const suite = try Suite.init(alloc, tc, false, if (msg_out) |msg| msg else @errorName(err)); - try results.append(suite); + try parser.eventInit(loadevt, "load", .{}); + _ = try parser.eventTargetDispatchEvent( + parser.toEventTarget(@TypeOf(runner.window), &runner.window), + loadevt, + ); + } - if (out == .text) { - std.debug.print("FAIL\t{s}\t{}\n", .{ tc, err }); + { + // wait for all async executions + var try_catch: Env.TryCatch = undefined; + try_catch.init(runner.scope); + defer try_catch.deinit(); + runner.loop.run() catch |err| { + if (try try_catch.err(arena)) |msg| { + err_out.* = msg; } - failures += 1; - continue; - } orelse { - // This test should _not_ have been run. - run -= 1; - continue; + return err; }; + } - const suite = try Suite.init(alloc, tc, true, res); - try results.append(suite); + // Check the final test status. + try runner.exec("report.status", "teststatus", err_out); - if (out == .json) { - continue; + // return the detailed result. + const res = try runner.eval("report.log", "report", err_out); + return try res.toString(arena); +} + +const Writer = struct { + format: Format, + arena: Allocator, + pass_count: usize = 0, + fail_count: usize = 0, + case_pass_count: usize = 0, + case_fail_count: usize = 0, + out: std.fs.File.Writer, + cases: std.ArrayListUnmanaged(Case) = .{}, + + const Format = enum { + json, + text, + summary, + }; + + fn init(arena: Allocator, format: Format) !Writer { + const out = std.io.getStdOut().writer(); + if (format == .json) { + try out.writeByte('['); } - if (!suite.pass) { - std.debug.print("Fail\t{s}\n{s}\n", .{ suite.name, suite.fmtMessage() }); - failures += 1; - } else { - std.debug.print("Pass\t{s}\n", .{suite.name}); + return .{ + .out = out, + .arena = arena, + .format = format, + }; + } + + fn finalize(self: *Writer) !void { + if (self.format == .json) { + try self.out.writeByte(']'); } + } - // display details - if (suite.cases) |cases| { - for (cases) |case| { - std.debug.print("\t{s}\t{s}\t{s}\n", .{ case.fmtStatus(), case.name, case.fmtMessage() }); + fn process(self: *Writer, test_file: []const u8, result_: ?[]const u8, err_: ?[]const u8) !void { + if (err_) |err| { + self.fail_count += 1; + switch (self.format) { + .text => return self.out.print("Fail\t{s}\n\t{s}\n", .{ test_file, err }), + .summary => return self.out.print("Fail 0/0\t{s}\n", .{test_file}), + .json => return std.json.stringify(Test{ + .pass = false, + .name = test_file, + .cases = &.{}, + }, .{ .whitespace = .indent_2 }, self.out), } + // just make sure we didn't fall through by mistake + unreachable; } - } - if (out == .json) { - var output = std.ArrayList(Test).init(alloc); - defer output.deinit(); - - for (results.items) |suite| { - var cases = std.ArrayList(Case).init(alloc); - defer cases.deinit(); - - if (suite.cases) |scases| { - for (scases) |case| { - try cases.append(Case{ - .pass = case.pass, - .name = case.name, - .message = case.message, - }); - } + // if we don't have an error, we must have a result + const result = result_ orelse return error.InvalidResult; + + var cases = &self.cases; + cases.clearRetainingCapacity(); // from previous run + + var pass = true; + var case_pass_count: usize = 0; + var case_fail_count: usize = 0; + + var lines = std.mem.splitScalar(u8, result, '\n'); + while (lines.next()) |line| { + if (line.len == 0) { + break; + } + var fields = std.mem.splitScalar(u8, line, '|'); + const case_name = fields.next() orelse { + std.debug.print("invalid result line: {s}\n", .{line}); + return error.InvalidResult; + }; + + const text_status = fields.next() orelse { + std.debug.print("invalid result line: {s}\n", .{line}); + return error.InvalidResult; + }; + + const case_pass = std.mem.eql(u8, text_status, "Pass"); + if (case_pass) { + case_pass_count += 1; } else { - // no cases, generate a fake one - try cases.append(Case{ - .pass = suite.pass, - .name = suite.name, - .message = suite.message, - }); + // If 1 case fails, we treat the entire file as a fail. + pass = false; + case_fail_count += 1; } - try output.append(Test{ - .pass = suite.pass, - .name = suite.name, - .cases = try cases.toOwnedSlice(), + try cases.append(self.arena, .{ + .name = case_name, + .pass = case_pass, + .message = fields.next(), }); } - defer { - for (output.items) |suite| { - alloc.free(suite.cases); - } + // our global counters + if (pass) { + self.pass_count += 1; + } else { + self.fail_count += 1; + } + self.case_pass_count += case_pass_count; + self.case_fail_count += case_fail_count; + + switch (self.format) { + .summary => try self.out.print("{s} {d}/{d}\t{s}\n", .{ statusText(pass), case_pass_count, case_pass_count + case_fail_count, test_file }), + .text => { + try self.out.print("{s}\t{s}\n", .{ statusText(pass), test_file }); + for (cases.items) |c| { + try self.out.print("\t{s}\t{s}\n", .{ statusText(c.pass), c.name }); + if (c.message) |msg| { + try self.out.print("\t\t{s}\n", .{msg}); + } + } + }, + .json => { + try std.json.stringify(Test{ + .pass = pass, + .name = test_file, + .cases = cases.items, + }, .{ .whitespace = .indent_2 }, self.out); + }, } - - try std.json.stringify(output.items, .{ .whitespace = .indent_2 }, std.io.getStdOut().writer()); - std.posix.exit(0); } - if (out == .text and failures > 0) { - std.debug.print("{d}/{d} tests suites failures\n", .{ failures, run }); - std.posix.exit(1); + fn statusText(pass: bool) []const u8 { + return if (pass) "Pass" else "Fail"; } -} - -// struct used for JSON output. -const Case = struct { - pass: bool, - name: []const u8, - message: ?[]const u8, }; -const Test = struct { - pass: bool, - crash: bool = false, - name: []const u8, - cases: []Case, + +const Command = struct { + format: Writer.Format, + filters: [][]const u8, }; -// shouldRun return true if the test should be run accroding to the given filters. -fn shouldRun(filter: [][]const u8, tc: []const u8) bool { - if (filter.len == 0) { - return true; - } +fn parseArgs(arena: Allocator) !Command { + const usage = + \\usage: {s} [options] [test filter] + \\ Run the Web Test Platform. + \\ + \\ -h, --help Print this help message and exit. + \\ --json result is formatted in JSON. + \\ --summary print a summary result. Incompatible w/ --json + \\ + ; + + var args = try std.process.argsWithAllocator(arena); + + // get the exec name. + const execname = args.next().?; + + var format = Writer.Format.text; + var filters: std.ArrayListUnmanaged([]const u8) = .{}; - for (filter) |f| { - if (std.mem.startsWith(u8, tc, f)) { - return true; + while (args.next()) |arg| { + if (std.mem.eql(u8, "-h", arg) or std.mem.eql(u8, "--help", arg)) { + try std.io.getStdErr().writer().print(usage, .{execname}); + std.posix.exit(0); } - if (std.mem.endsWith(u8, tc, f)) { - return true; + + if (std.mem.eql(u8, "--json", arg)) { + format = .json; + } else if (std.mem.eql(u8, "--summary", arg)) { + format = .summary; + } else { + try filters.append(arena, arg); } } - return false; -} -// runSafe rune each test cae in a separate child process to detect crashes. -fn runSafe( - allocator: std.mem.Allocator, - execname: []const u8, - out: Out, - testcases: [][]const u8, - filter: [][]const u8, -) !void { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const alloc = arena.allocator(); - - const Result = enum { - success, - crash, + return .{ + .format = format, + .filters = filters.items, }; +} - var argv = try std.ArrayList([]const u8).initCapacity(alloc, 3); - defer argv.deinit(); - argv.appendAssumeCapacity(execname); - // always require json output to count test cases results - argv.appendAssumeCapacity("--json"); +const TestIterator = struct { + dir: Dir, + walker: Dir.Walker, + filters: [][]const u8, - var output = std.ArrayList(Test).init(alloc); + const Dir = std.fs.Dir; - for (testcases) |tc| { - if (!shouldRun(filter, tc)) { - continue; - } + fn init(arena: Allocator, root: []const u8, filters: [][]const u8) !TestIterator { + var dir = try std.fs.cwd().openDir(root, .{ .iterate = true, .no_follow = true }); + errdefer dir.close(); - // append the test case to argv and pop it before next loop. - argv.appendAssumeCapacity(tc); - defer _ = argv.pop(); + return .{ + .dir = dir, + .filters = filters, + .walker = try dir.walk(arena), + }; + } - const run = try std.process.Child.run(.{ - .allocator = alloc, - .argv = argv.items, - .max_output_bytes = 1024 * 1024, - }); + fn deinit(self: *TestIterator) void { + self.dir.close(); + } - const result: Result = switch (run.term) { - .Exited => .success, - else => .crash, - }; + fn next(self: *TestIterator) !?[]const u8 { + NEXT: while (try self.walker.next()) |entry| { + if (entry.kind != .file) { + continue; + } - // read the JSON result from stdout - var tests: []Test = undefined; - if (result != .crash) { - const parsed = try std.json.parseFromSlice([]Test, alloc, run.stdout, .{}); - tests = parsed.value; - } + if (std.mem.startsWith(u8, entry.path, "resources/")) { + // resources for running the tests themselves, not actual tests + continue; + } - // summary display - if (out == .summary) { - defer std.debug.print("\t{s}\n", .{tc}); - if (result == .crash) { - std.debug.print("Crash\t", .{}); + if (!std.mem.endsWith(u8, entry.basename, ".html") and !std.mem.endsWith(u8, entry.basename, ".htm")) { continue; } - // count results - var pass: u32 = 0; - var all: u32 = 0; - for (tests) |ttc| { - for (ttc.cases) |c| { - all += 1; - if (c.pass) pass += 1; + const path = entry.path; + for (self.filters) |filter| { + if (std.mem.indexOf(u8, path, filter) == null) { + continue :NEXT; } } - const status = if (all > 0 and pass == all) "Pass" else "Fail"; - std.debug.print("{s} {d}/{d}", .{ status, pass, all }); - continue; + return path; } - // json display - if (out == .json) { - if (result == .crash) { - var cases = [_]Case{.{ - .pass = false, - .name = "crash", - .message = run.stderr, - }}; - try output.append(Test{ - .pass = false, - .crash = true, - .name = tc, - .cases = cases[0..1], - }); - continue; - } + return null; + } +}; - try output.appendSlice(tests); - continue; - } +const Case = struct { + pass: bool, + name: []const u8, + message: ?[]const u8, +}; - // normal display - std.debug.print("{s}\n", .{tc}); - if (result == .crash) { - std.debug.print("Crash\n{s}", .{run.stderr}); - continue; - } - var pass: u32 = 0; - var all: u32 = 0; - for (tests) |ttc| { - for (ttc.cases) |c| { - const status = if (c.pass) "Pass" else "Fail"; - std.debug.print("{s}\t{s}\n", .{ status, c.name }); - all += 1; - if (c.pass) pass += 1; - } +const Test = struct { + pass: bool, + crash: bool = false, + name: []const u8, + cases: []Case, +}; + +pub const FileLoader = struct { + path: []const u8, + arena: Allocator, + files: std.StringHashMapUnmanaged([]const u8), + + pub fn init(arena: Allocator, path: []const u8) FileLoader { + return .{ + .path = path, + .files = .{}, + .arena = arena, + }; + } + pub fn get(self: *FileLoader, name: []const u8) ![]const u8 { + const gop = try self.files.getOrPut(self.arena, name); + if (gop.found_existing == false) { + gop.key_ptr.* = try self.arena.dupe(u8, name); + gop.value_ptr.* = self.load(name) catch |err| { + _ = self.files.remove(name); + return err; + }; } - const status = if (all > 0 and pass == all) "Pass" else "Fail"; - std.debug.print("{s} {d}/{d}\n\n", .{ status, pass, all }); + return gop.value_ptr.*; } - if (out == .json) { - try std.json.stringify(output.items, .{ .whitespace = .indent_2 }, std.io.getStdOut().writer()); + fn load(self: *FileLoader, name: []const u8) ![]const u8 { + const filename = try std.fs.path.join(self.arena, &.{ self.path, name }); + var file = try std.fs.cwd().openFile(filename, .{}); + defer file.close(); + + return file.readToEndAlloc(self.arena, 4 * 1024 * 1024); } -} +}; diff --git a/src/wpt/fileloader.zig b/src/wpt/fileloader.zig deleted file mode 100644 index edbbab6b9..000000000 --- a/src/wpt/fileloader.zig +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright (C) 2023-2024 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const std = @import("std"); -const fspath = std.fs.path; - -// FileLoader loads files content from the filesystem. -pub const FileLoader = struct { - const FilesMap = std.StringHashMap([]const u8); - - files: FilesMap, - path: []const u8, - alloc: std.mem.Allocator, - - pub fn init(alloc: std.mem.Allocator, path: []const u8) FileLoader { - const files = FilesMap.init(alloc); - - return FileLoader{ - .path = path, - .alloc = alloc, - .files = files, - }; - } - pub fn get(self: *FileLoader, name: []const u8) ![]const u8 { - if (!self.files.contains(name)) { - try self.load(name); - } - return self.files.get(name).?; - } - pub fn load(self: *FileLoader, name: []const u8) !void { - const filename = try fspath.join(self.alloc, &.{ self.path, name }); - defer self.alloc.free(filename); - var file = try std.fs.cwd().openFile(filename, .{}); - defer file.close(); - - const file_size = try file.getEndPos(); - const content = try file.readToEndAlloc(self.alloc, file_size); - const namedup = try self.alloc.dupe(u8, name); - try self.files.put(namedup, content); - } - pub fn deinit(self: *FileLoader) void { - var iter = self.files.iterator(); - while (iter.next()) |entry| { - self.alloc.free(entry.key_ptr.*); - self.alloc.free(entry.value_ptr.*); - } - self.files.deinit(); - } -}; diff --git a/src/wpt/run.zig b/src/wpt/run.zig deleted file mode 100644 index a03789736..000000000 --- a/src/wpt/run.zig +++ /dev/null @@ -1,151 +0,0 @@ -// Copyright (C) 2023-2024 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const std = @import("std"); -const fspath = std.fs.path; -const Allocator = std.mem.Allocator; - -const Env = @import("../browser/env.zig").Env; -const FileLoader = @import("fileloader.zig").FileLoader; -const Window = @import("../browser/html/window.zig").Window; - -const parser = @import("../browser/netsurf.zig"); -const polyfill = @import("../browser/polyfill/polyfill.zig"); - -// runWPT parses the given HTML file, starts a js env and run the first script -// tags containing javascript sources. -// It loads first the js libs files. -pub fn run(arena: Allocator, comptime dir: []const u8, f: []const u8, loader: *FileLoader, err_msg: *?[]const u8) !?[]const u8 { - // document - const html = blk: { - const file = try std.fs.cwd().openFile(f, .{}); - defer file.close(); - break :blk try file.readToEndAlloc(arena, 128 * 1024); - }; - - if (std.mem.indexOf(u8, html, "testharness.js") == null) { - // This isn't a test. A lot of files are helpers/content for tests to - // make use of. - return null; - } - - const dirname = fspath.dirname(f[dir.len..]) orelse unreachable; - - var runner = try @import("../testing.zig").jsRunner(arena, .{ - .html = html, - }); - defer runner.deinit(); - try polyfill.load(arena, runner.scope); - - // display console logs - defer { - const res = runner.eval("console.join('\\n');", "console", err_msg) catch unreachable; - const log = res.toString(arena) catch unreachable; - if (log.len > 0) { - std.debug.print("-- CONSOLE LOG\n{s}\n--\n", .{log}); - } - } - - try runner.exec( - \\ console = []; - \\ console.log = function () { - \\ console.push(...arguments); - \\ }; - \\ console.debug = function () { - \\ console.push("debug", ...arguments); - \\ }; - , "init", err_msg); - - // loop over the scripts. - const doc = parser.documentHTMLToDocument(runner.state.document.?); - const scripts = try parser.documentGetElementsByTagName(doc, "script"); - const slen = try parser.nodeListLength(scripts); - for (0..slen) |i| { - const s = (try parser.nodeListItem(scripts, @intCast(i))).?; - - // If the script contains an src attribute, load it. - if (try parser.elementGetAttribute(@as(*parser.Element, @ptrCast(s)), "src")) |src| { - var path = src; - if (!std.mem.startsWith(u8, src, "/")) { - // no need to free path, thanks to the arena. - path = try fspath.join(arena, &.{ "/", dirname, path }); - } - try runner.exec(try loader.get(path), src, err_msg); - } - - // If the script as a source text, execute it. - const src = try parser.nodeTextContent(s) orelse continue; - try runner.exec(src, null, err_msg); - } - - // Mark tests as ready to run. - const loadevt = try parser.eventCreate(); - defer parser.eventDestroy(loadevt); - - try parser.eventInit(loadevt, "load", .{}); - _ = try parser.eventTargetDispatchEvent( - parser.toEventTarget(@TypeOf(runner.window), &runner.window), - loadevt, - ); - - // wait for all async executions - { - var try_catch: Env.TryCatch = undefined; - try_catch.init(runner.scope); - defer try_catch.deinit(); - runner.loop.run() catch |err| { - if (try try_catch.err(arena)) |msg| { - err_msg.* = msg; - } - return err; - }; - } - - // Check the final test status. - try runner.exec("report.status", "teststatus", err_msg); - - // return the detailed result. - const res = try runner.eval("report.log", "report", err_msg); - return try res.toString(arena); -} - -// browse the path to find the tests list. -pub fn find(allocator: Allocator, comptime path: []const u8, list: *std.ArrayList([]const u8)) !void { - var dir = try std.fs.cwd().openDir(path, .{ .iterate = true, .no_follow = true }); - defer dir.close(); - - var walker = try dir.walk(allocator); - defer walker.deinit(); - - while (try walker.next()) |entry| { - if (entry.kind != .file) { - continue; - } - - if (std.mem.startsWith(u8, entry.path, "resources/")) { - // resources for running the tests themselves, not actual tests - continue; - } - - if (!std.mem.endsWith(u8, entry.basename, ".html") and !std.mem.endsWith(u8, entry.basename, ".htm")) { - continue; - } - - try list.append(try fspath.join(allocator, &.{ path, entry.path })); - } -} diff --git a/src/wpt/testcase.zig b/src/wpt/testcase.zig deleted file mode 100644 index 1f3e09153..000000000 --- a/src/wpt/testcase.zig +++ /dev/null @@ -1,263 +0,0 @@ -// Copyright (C) 2023-2024 Lightpanda (Selecy SAS) -// -// Francis Bouvier -// Pierre Tachoire -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as -// published by the Free Software Foundation, either version 3 of the -// License, or (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -const std = @import("std"); -const testing = std.testing; - -pub const Case = struct { - pass: bool, - name: []const u8, - - message: ?[]const u8, - - fn init(alloc: std.mem.Allocator, name: []const u8, status: []const u8, message: []const u8) !Case { - var case = Case{ - .pass = std.mem.eql(u8, "Pass", status), - .name = try alloc.dupe(u8, name), - .message = null, - }; - - if (message.len > 0) { - case.message = try alloc.dupe(u8, message); - } - - return case; - } - - fn deinit(self: Case, alloc: std.mem.Allocator) void { - alloc.free(self.name); - - if (self.message) |msg| { - alloc.free(msg); - } - } - - pub fn fmtStatus(self: Case) []const u8 { - if (self.pass) { - return "Pass"; - } - return "Fail"; - } - - pub fn fmtMessage(self: Case) []const u8 { - if (self.message) |v| { - return v; - } - return ""; - } -}; - -pub const Suite = struct { - alloc: std.mem.Allocator, - pass: bool, - name: []const u8, - message: ?[]const u8, - cases: ?[]Case, - - // caller owns the wpt.Suite. - // owner must call deinit(). - pub fn init(alloc: std.mem.Allocator, name: []const u8, pass: bool, res: []const u8) !Suite { - var suite = Suite{ - .alloc = alloc, - .pass = false, - .name = try alloc.dupe(u8, name), - .message = null, - .cases = null, - }; - - // handle JS error. - if (!pass) { - suite.message = try alloc.dupe(u8, res); - return suite; - } - - // no JS error, let's try to parse the result. - suite.pass = true; - - // special case: the result contains only "Pass" message - if (std.mem.eql(u8, "Pass", res)) { - return suite; - } - - var cases = std.ArrayList(Case).init(alloc); - defer cases.deinit(); - - var lines = std.mem.splitScalar(u8, res, '\n'); - while (lines.next()) |line| { - if (line.len == 0) { - break; - } - var fields = std.mem.splitScalar(u8, line, '|'); - var ff: [3][]const u8 = .{ "", "", "" }; - var i: u8 = 0; - while (fields.next()) |field| { - if (i >= 3) { - suite.pass = false; - suite.message = try alloc.dupe(u8, res); - return suite; - } - - ff[i] = field; - i += 1; - } - - // invalid output format - if (i != 2 and i != 3) { - suite.pass = false; - suite.message = try alloc.dupe(u8, res); - return suite; - } - - const case = try Case.init(alloc, ff[0], ff[1], ff[2]); - if (!case.pass) { - suite.pass = false; - } - - try cases.append(case); - } - - if (cases.items.len == 0) { - // no test case, create a failed one. - suite.pass = false; - try cases.append(.{ - .pass = false, - .name = "no test case", - .message = "no test case", - }); - } - - suite.cases = try cases.toOwnedSlice(); - - return suite; - } - - pub fn deinit(self: Suite) void { - self.alloc.free(self.name); - - if (self.message) |res| { - self.alloc.free(res); - } - - if (self.cases) |cases| { - for (cases) |case| { - case.deinit(self.alloc); - } - self.alloc.free(cases); - } - } - - pub fn fmtMessage(self: Suite) []const u8 { - if (self.message) |v| { - return v; - } - return ""; - } -}; - -test "success test case" { - const alloc = testing.allocator; - - const Res = struct { - pass: bool, - result: []const u8, - }; - - const res = Res{ - .pass = true, - .result = - \\Empty string as a name for Document.getElementsByTagName|Pass - \\Empty string as a name for Element.getElementsByTagName|Pass - \\ - , - }; - - const suite = Suite.init(alloc, "foo", res.pass, res.result) catch unreachable; // TODO - defer suite.deinit(); - - try testing.expect(suite.pass == true); - try testing.expect(suite.cases != null); - try testing.expect(suite.cases.?.len == 2); - try testing.expect(suite.cases.?[0].pass == true); - try testing.expect(suite.cases.?[1].pass == true); -} - -test "failed test case" { - const alloc = testing.allocator; - - const Res = struct { - pass: bool, - result: []const u8, - }; - - const res = Res{ - .pass = true, - .result = - \\Empty string as a name for Document.getElementsByTagName|Pass - \\Empty string as a name for Element.getElementsByTagName|Fail|div.getElementsByTagName is not a function - \\ - , - }; - - const suite = Suite.init(alloc, "foo", res.pass, res.result) catch unreachable; // TODO - defer suite.deinit(); - - try testing.expect(suite.pass == false); - try testing.expect(suite.cases != null); - try testing.expect(suite.cases.?.len == 2); - try testing.expect(suite.cases.?[0].pass == true); - try testing.expect(suite.cases.?[1].pass == false); -} - -test "invalid result" { - const alloc = testing.allocator; - - const Res = struct { - pass: bool, - result: []const u8, - }; - - const res = Res{ - .pass = true, - .result = - \\this is|an|invalid|result - , - }; - - const suite = Suite.init(alloc, "foo", res.pass, res.result) catch unreachable; // TODO - defer suite.deinit(); - - try testing.expect(suite.pass == false); - try testing.expect(suite.message != null); - try testing.expect(std.mem.eql(u8, res.result, suite.message.?)); - try testing.expect(suite.cases == null); - - const res2 = Res{ - .pass = true, - .result = - \\this is an invalid result. - , - }; - - const suite2 = Suite.init(alloc, "foo", res2.pass, res2.result) catch unreachable; // TODO - defer suite2.deinit(); - - try testing.expect(suite2.pass == false); - try testing.expect(suite2.message != null); - try testing.expect(std.mem.eql(u8, res2.result, suite2.message.?)); - try testing.expect(suite2.cases == null); -} From 3a8c3acd8215d6ec3b3e92241c1d0db0f0c1d9e2 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 16:02:04 +0800 Subject: [PATCH 2/8] Update workflows, add summary Remove --safe option from WPT workflows (it's no longer valid) Include a total test/case summary when --summary or --text (default) is used. --- .github/workflows/wpt.yml | 4 ++-- src/main_wpt.zig | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wpt.yml b/.github/workflows/wpt.yml index 349c94c18..d90ca5061 100644 --- a/.github/workflows/wpt.yml +++ b/.github/workflows/wpt.yml @@ -55,7 +55,7 @@ jobs: - uses: ./.github/actions/install - - run: zig build wpt -- --safe --summary + - run: zig build wpt -- --summary # For now WPT tests doesn't pass at all. # We accept then to continue the job on failure. @@ -80,7 +80,7 @@ jobs: - uses: ./.github/actions/install - name: json output - run: zig build wpt -- --safe --json > wpt.json + run: zig build wpt -- --json > wpt.json - name: write commit run: | diff --git a/src/main_wpt.zig b/src/main_wpt.zig index 9a70974a6..1604b849e 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -203,6 +203,13 @@ const Writer = struct { fn finalize(self: *Writer) !void { if (self.format == .json) { try self.out.writeByte(']'); + } else { + try self.out.print("\n==Summary==\nTests: {d}/{d}\nCases: {d}/{d}\n", .{ + self.pass_count, + self.pass_count + self.fail_count, + self.case_pass_count, + self.case_pass_count + self.case_fail_count, + }); } } From ace5ba57ed1cf46ca25ed1e45e72f8465f6a444e Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 16:06:57 +0800 Subject: [PATCH 3/8] remove wpt --safe flag from Makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 13c61348b..4fb54cda8 100644 --- a/Makefile +++ b/Makefile @@ -85,11 +85,11 @@ shell: ## Run WPT tests wpt: @printf "\e[36mBuilding wpt...\e[0m\n" - @$(ZIG) build wpt -- --safe $(filter-out $@,$(MAKECMDGOALS)) || (printf "\e[33mBuild ERROR\e[0m\n"; exit 1;) + @$(ZIG) build wpt -- $(filter-out $@,$(MAKECMDGOALS)) || (printf "\e[33mBuild ERROR\e[0m\n"; exit 1;) wpt-summary: @printf "\e[36mBuilding wpt...\e[0m\n" - @$(ZIG) build wpt -- --safe --summary $(filter-out $@,$(MAKECMDGOALS)) || (printf "\e[33mBuild ERROR\e[0m\n"; exit 1;) + @$(ZIG) build wpt -- --summary $(filter-out $@,$(MAKECMDGOALS)) || (printf "\e[33mBuild ERROR\e[0m\n"; exit 1;) ## Test test: From 9dadc1b28c3c16a8ed09a9e13670683d284d46de Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 16:42:08 +0800 Subject: [PATCH 4/8] handle tests in the root of the test folder --- src/main_wpt.zig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main_wpt.zig b/src/main_wpt.zig index 1604b849e..6dce782b4 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -104,7 +104,8 @@ fn run(arena: Allocator, test_file: []const u8, loader: *FileLoader, err_out: *? return null; } - const dirname = std.fs.path.dirname(test_file) orelse unreachable; + // this returns null for the success.html test in the root of tests/wpt + const dirname = std.fs.path.dirname(test_file) orelse ""; var runner = try @import("testing.zig").jsRunner(arena, .{ .html = html, From 9775cebdf8340dd476c214584d3ce9d7866ef16d Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 19:35:00 +0800 Subject: [PATCH 5/8] Fix a couple possible segfaults base on strange usage (WPT stuff) --- src/browser/netsurf.zig | 8 ++++++++ vendor/netsurf/libdom | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/browser/netsurf.zig b/src/browser/netsurf.zig index eb28fd0e4..d20b37616 100644 --- a/src/browser/netsurf.zig +++ b/src/browser/netsurf.zig @@ -1626,6 +1626,14 @@ pub fn elementHTMLGetTagType(elem_html: *ElementHTML) !Tag { var tag_type: c.dom_html_element_type = undefined; const err = elementHTMLVtable(elem_html).dom_html_element_get_tag_type.?(elem_html, &tag_type); try DOMErr(err); + + if (tag_type >= @intFromEnum(Tag.undef)) { + // This is questionable, but std.meta.intToEnum has more overhead + // Added this because this WPT test started to fail once we + // introduced an SVGElement: + // html/dom/documents/dom-tree-accessors/document.title-09.html + return Tag.undef; + } return @as(Tag, @enumFromInt(tag_type)); } diff --git a/vendor/netsurf/libdom b/vendor/netsurf/libdom index c81dfc300..b2c17b147 160000 --- a/vendor/netsurf/libdom +++ b/vendor/netsurf/libdom @@ -1 +1 @@ -Subproject commit c81dfc300b47965ec2c43a902bdc9ef736c1ab7d +Subproject commit b2c17b1476d1bb273d9e92eae32ae576998465cf From 78031c5a7fd9704dcc5b8b58093924d2851050ba Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 20:17:48 +0800 Subject: [PATCH 6/8] generate proper JSON --- src/main_wpt.zig | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main_wpt.zig b/src/main_wpt.zig index 6dce782b4..b749559ae 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -203,7 +203,12 @@ const Writer = struct { fn finalize(self: *Writer) !void { if (self.format == .json) { - try self.out.writeByte(']'); + // When we write a test output, we add a trailing comma to act as + // a separator for the next test. We need to add this dummy entry + // to make it valid json. + // Better option could be to change the formatter to work on JSONL: + // https://github.com/lightpanda-io/perf-fmt/blob/main/wpt/wpt.go + try self.out.writeAll("{\"name\":\"trailing-hack\",\"pass\": true}]"); } else { try self.out.print("\n==Summary==\nTests: {d}/{d}\nCases: {d}/{d}\n", .{ self.pass_count, @@ -298,6 +303,8 @@ const Writer = struct { .name = test_file, .cases = cases.items, }, .{ .whitespace = .indent_2 }, self.out); + // separator, see `finalize` for the hack we use to terminate this + try self.out.writeByte(','); }, } } From 2f1f87066b084f0843686402719f41b7d39aa318 Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 21:07:46 +0800 Subject: [PATCH 7/8] generate proper JSON (for real this time?) --- src/main_wpt.zig | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main_wpt.zig b/src/main_wpt.zig index b749559ae..2f3e93243 100644 --- a/src/main_wpt.zig +++ b/src/main_wpt.zig @@ -225,11 +225,14 @@ const Writer = struct { switch (self.format) { .text => return self.out.print("Fail\t{s}\n\t{s}\n", .{ test_file, err }), .summary => return self.out.print("Fail 0/0\t{s}\n", .{test_file}), - .json => return std.json.stringify(Test{ - .pass = false, - .name = test_file, - .cases = &.{}, - }, .{ .whitespace = .indent_2 }, self.out), + .json => { + try std.json.stringify(Test{ + .pass = false, + .name = test_file, + .cases = &.{}, + }, .{ .whitespace = .indent_2 }, self.out); + return self.out.writeByte(','); + }, } // just make sure we didn't fall through by mistake unreachable; From a12708f8e0fcfd12ef2407d8ed77a70486837c2b Mon Sep 17 00:00:00 2001 From: Karl Seguin Date: Fri, 2 May 2025 21:33:55 +0800 Subject: [PATCH 8/8] fix tag type check --- src/browser/netsurf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/browser/netsurf.zig b/src/browser/netsurf.zig index d20b37616..263c204a4 100644 --- a/src/browser/netsurf.zig +++ b/src/browser/netsurf.zig @@ -1627,7 +1627,7 @@ pub fn elementHTMLGetTagType(elem_html: *ElementHTML) !Tag { const err = elementHTMLVtable(elem_html).dom_html_element_get_tag_type.?(elem_html, &tag_type); try DOMErr(err); - if (tag_type >= @intFromEnum(Tag.undef)) { + if (tag_type >= 255) { // This is questionable, but std.meta.intToEnum has more overhead // Added this because this WPT test started to fail once we // introduced an SVGElement: