test: rework how filtering works

 * make test names contain the fully qualified name
 * make test filters match the fully qualified name
 * allow multiple test filters, where a test is skipped if it does not
   match any of the specified filters

Author: jacobly0

build.zig

@@ -390,7 +390,7 @@ pub fn build(b: *std.Build) !void {
         }
     }
 
-    const test_filter = b.option([]const u8, "test-filter", "Skip tests that do not match filter");
+    const test_filters = b.option([]const []const u8, "test-filter", "Skip tests that do not match any filter") orelse &[0][]const u8{};
 
     const test_cases_options = b.addOptions();
     check_case_exe.root_module.addOptions("build_options", test_cases_options);
@@ -418,7 +418,7 @@ pub fn build(b: *std.Build) !void {
     test_cases_options.addOption(?[]const u8, "glibc_runtimes_dir", b.glibc_runtimes_dir);
     test_cases_options.addOption([:0]const u8, "version", version);
     test_cases_options.addOption(std.SemanticVersion, "semver", semver);
-    test_cases_options.addOption(?[]const u8, "test_filter", test_filter);
+    test_cases_options.addOption([]const []const u8, "test_filters", test_filters);
 
     var chosen_opt_modes_buf: [4]builtin.OptimizeMode = undefined;
     var chosen_mode_index: usize = 0;
@@ -454,7 +454,7 @@ pub fn build(b: *std.Build) !void {
     }).step);
 
     const test_cases_step = b.step("test-cases", "Run the main compiler test cases");
-    try tests.addCases(b, test_cases_step, test_filter, check_case_exe, .{
+    try tests.addCases(b, test_cases_step, test_filters, check_case_exe, .{
         .enable_llvm = enable_llvm,
         .llvm_has_m68k = llvm_has_m68k,
         .llvm_has_csky = llvm_has_csky,
@@ -464,7 +464,7 @@ pub fn build(b: *std.Build) !void {
     test_step.dependOn(test_cases_step);
 
     test_step.dependOn(tests.addModuleTests(b, .{
-        .test_filter = test_filter,
+        .test_filters = test_filters,
         .root_src = "test/behavior.zig",
         .name = "behavior",
         .desc = "Run the behavior tests",
@@ -477,7 +477,7 @@ pub fn build(b: *std.Build) !void {
     }));
 
     test_step.dependOn(tests.addModuleTests(b, .{
-        .test_filter = test_filter,
+        .test_filters = test_filters,
         .root_src = "test/c_import.zig",
         .name = "c-import",
         .desc = "Run the @cImport tests",
@@ -489,7 +489,7 @@ pub fn build(b: *std.Build) !void {
     }));
 
     test_step.dependOn(tests.addModuleTests(b, .{
-        .test_filter = test_filter,
+        .test_filters = test_filters,
         .root_src = "lib/compiler_rt.zig",
         .name = "compiler-rt",
         .desc = "Run the compiler_rt tests",
@@ -501,7 +501,7 @@ pub fn build(b: *std.Build) !void {
     }));
 
     test_step.dependOn(tests.addModuleTests(b, .{
-        .test_filter = test_filter,
+        .test_filters = test_filters,
         .root_src = "lib/c.zig",
         .name = "universal-libc",
         .desc = "Run the universal libc tests",
@@ -512,7 +512,7 @@ pub fn build(b: *std.Build) !void {
         .skip_libc = true,
     }));
 
-    test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));
+    test_step.dependOn(tests.addCompareOutputTests(b, test_filters, optimization_modes));
     test_step.dependOn(tests.addStandaloneTests(
         b,
         optimization_modes,
@@ -523,16 +523,16 @@ pub fn build(b: *std.Build) !void {
     ));
     test_step.dependOn(tests.addCAbiTests(b, skip_non_native, skip_release));
     test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, enable_ios_sdk, false, enable_symlinks_windows));
-    test_step.dependOn(tests.addStackTraceTests(b, test_filter, optimization_modes));
+    test_step.dependOn(tests.addStackTraceTests(b, test_filters, optimization_modes));
     test_step.dependOn(tests.addCliTests(b));
-    test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, optimization_modes));
-    test_step.dependOn(tests.addTranslateCTests(b, test_filter));
+    test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filters, optimization_modes));
+    test_step.dependOn(tests.addTranslateCTests(b, test_filters));
     if (!skip_run_translated_c) {
-        test_step.dependOn(tests.addRunTranslatedCTests(b, test_filter, target));
+        test_step.dependOn(tests.addRunTranslatedCTests(b, test_filters, target));
     }
 
     test_step.dependOn(tests.addModuleTests(b, .{
-        .test_filter = test_filter,
+        .test_filters = test_filters,
         .root_src = "lib/std/std.zig",
         .name = "std",
         .desc = "Run the standard library tests",

doc/langref.html.in

@@ -988,13 +988,13 @@ fn addOne(number: i32) i32 {
         printed to standard error by the default test runner:
       </p>
       <dl>
-        <dt><samp>Test [1/2] test.expect addOne adds one to 41...</samp></dt>
+        <dt><samp>1/2 testing_introduction.test.expect addOne adds one to 41...</samp></dt>
         <dd>Lines like this indicate which test, out of the total number of tests, is being run.
-          In this case, <samp>[1/2]</samp> indicates that the first test, out of a total of
-          two test, is being run. Note that, when the test runner program's standard error is output
+          In this case, <samp>1/2</samp> indicates that the first test, out of a total of two tests,
+          is being run. Note that, when the test runner program's standard error is output
           to the terminal, these lines are cleared when a test succeeds.
         </dd>
-        <dt><samp>Test [2/2] decltest.addOne...</samp></dt>
+        <dt><samp>2/2 testing_introduction.decltest.addOne...</samp></dt>
         <dd>When the test name is an identifier, the default test runner uses the text
           decltest instead of test.
         </dd>

lib/std/Build.zig

@@ -855,7 +855,9 @@ pub const TestOptions = struct {
     optimize: std.builtin.OptimizeMode = .Debug,
     version: ?std.SemanticVersion = null,
     max_rss: usize = 0,
+    /// deprecated: use `.filters = &.{filter}` instead of `.filter = filter`.
     filter: ?[]const u8 = null,
+    filters: []const []const u8 = &.{},
     test_runner: ?[]const u8 = null,
     link_libc: ?bool = null,
     single_threaded: ?bool = null,
@@ -888,7 +890,12 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile {
             .error_tracing = options.error_tracing,
         },
         .max_rss = options.max_rss,
-        .filter = options.filter,
+        .filters = if (options.filter != null and options.filters.len > 0) filters: {
+            const filters = b.allocator.alloc([]const u8, 1 + options.filters.len) catch @panic("OOM");
+            filters[0] = b.dupe(options.filter.?);
+            for (filters[1..], options.filters) |*dest, source| dest.* = b.dupe(source);
+            break :filters filters;
+        } else b.dupeStrings(if (options.filter) |filter| &.{filter} else options.filters),
         .test_runner = options.test_runner,
         .use_llvm = options.use_llvm,
         .use_lld = options.use_lld,
@@ -993,9 +1000,7 @@ pub fn dupe(self: *Build, bytes: []const u8) []u8 {
 /// Duplicates an array of strings without the need to handle out of memory.
 pub fn dupeStrings(self: *Build, strings: []const []const u8) [][]u8 {
     const array = self.allocator.alloc([]u8, strings.len) catch @panic("OOM");
-    for (strings, 0..) |s, i| {
-        array[i] = self.dupe(s);
-    }
+    for (array, strings) |*dest, source| dest.* = self.dupe(source);
     return array;
 }
 

lib/std/Build/Step/Compile.zig

@@ -54,7 +54,7 @@ global_base: ?u64 = null,
 /// Set via options; intended to be read-only after that.
 zig_lib_dir: ?LazyPath,
 exec_cmd_args: ?[]const ?[]const u8,
-filter: ?[]const u8,
+filters: []const []const u8,
 test_runner: ?[]const u8,
 test_server_mode: bool,
 wasi_exec_model: ?std.builtin.WasiExecModel = null,
@@ -223,7 +223,7 @@ pub const Options = struct {
     linkage: ?Linkage = null,
     version: ?std.SemanticVersion = null,
     max_rss: usize = 0,
-    filter: ?[]const u8 = null,
+    filters: []const []const u8 = &.{},
     test_runner: ?[]const u8 = null,
     use_llvm: ?bool = null,
     use_lld: ?bool = null,
@@ -310,7 +310,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile {
         .installed_headers = ArrayList(*Step).init(owner.allocator),
         .zig_lib_dir = null,
         .exec_cmd_args = null,
-        .filter = options.filter,
+        .filters = options.filters,
         .test_runner = options.test_runner,
         .test_server_mode = options.test_runner == null,
         .rdynamic = false,
@@ -1297,7 +1297,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
         try zig_args.append(b.fmt("0x{x}", .{image_base}));
     }
 
-    if (self.filter) |filter| {
+    for (self.filters) |filter| {
         try zig_args.append("--test-filter");
         try zig_args.append(filter);
     }

src/Compilation.zig

@@ -217,7 +217,7 @@ libcxx_abi_version: libcxx.AbiVersion = libcxx.AbiVersion.default,
 /// This mutex guards all `Compilation` mutable state.
 mutex: std.Thread.Mutex = .{},
 
-test_filter: ?[]const u8,
+test_filters: []const []const u8,
 test_name_prefix: ?[]const u8,
 
 emit_asm: ?EmitLoc,
@@ -1097,7 +1097,7 @@ pub const CreateOptions = struct {
     native_system_include_paths: []const []const u8 = &.{},
     clang_preprocessor_mode: ClangPreprocessorMode = .no,
     reference_trace: ?u32 = null,
-    test_filter: ?[]const u8 = null,
+    test_filters: []const []const u8 = &.{},
     test_name_prefix: ?[]const u8 = null,
     test_runner_path: ?[]const u8 = null,
     subsystem: ?std.Target.SubSystem = null,
@@ -1506,7 +1506,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
             .formatted_panics = formatted_panics,
             .time_report = options.time_report,
             .stack_report = options.stack_report,
-            .test_filter = options.test_filter,
+            .test_filters = options.test_filters,
             .test_name_prefix = options.test_name_prefix,
             .debug_compiler_runtime_libs = options.debug_compiler_runtime_libs,
             .debug_compile_errors = options.debug_compile_errors,
@@ -1613,7 +1613,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
                 hash.add(options.config.use_lib_llvm);
                 hash.add(options.config.dll_export_fns);
                 hash.add(options.config.is_test);
-                hash.addOptionalBytes(options.test_filter);
+                hash.addListOfBytes(options.test_filters);
                 hash.addOptionalBytes(options.test_name_prefix);
                 hash.add(options.skip_linker_dependencies);
                 hash.add(formatted_panics);
@@ -2475,7 +2475,7 @@ fn addNonIncrementalStuffToCacheManifest(
         try addModuleTableToCacheHash(gpa, arena, &man.hash, mod.root_mod, mod.main_mod, .{ .files = man });
 
         // Synchronize with other matching comments: ZigOnlyHashStuff
-        man.hash.addOptionalBytes(comp.test_filter);
+        man.hash.addListOfBytes(comp.test_filters);
         man.hash.addOptionalBytes(comp.test_name_prefix);
         man.hash.add(comp.skip_linker_dependencies);
         man.hash.add(comp.formatted_panics);

src/InternPool.zig

@@ -7904,7 +7904,7 @@ pub fn destroyNamespace(ip: *InternPool, gpa: Allocator, index: NamespaceIndex)
     ip.namespacePtr(index).* = .{
         .parent = undefined,
         .file_scope = undefined,
-        .ty = undefined,
+        .decl_index = undefined,
     };
     ip.namespaces_free_list.append(gpa, index) catch {
         // In order to keep `destroyNamespace` a non-fallible function, we ignore memory