From 7144ede537fd0f990cc20ae01c8200497c769ee1 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 3 Sep 2025 15:29:55 -0400 Subject: [PATCH 1/2] [libc++] Make it possible to mark a gen-test as UNSUPPORTED Previously, only the tests that are generated by the gen-test could be marked as UNSUPPORTED. After this patch, the gen-test itself can be marked as UNSUPPORTED, which makes it possible to add RUN lines that would be an error when unsupported to the gen-test itself. --- .../clang/clang_modules_include.gen.py | 78 ++++++------------- .../test/extensions/posix/xopen_source.gen.py | 25 +++--- libcxx/test/libcxx-03/clang_tidy.gen.py | 20 ++--- libcxx/test/libcxx/clang_tidy.gen.py | 14 ++-- libcxx/test/libcxx/transitive_includes.gen.py | 2 - .../test/selftest/gen.cpp/unsupported.gen.cpp | 18 +++++ libcxx/test/std/double_include.gen.py | 8 +- libcxx/test/std/header_inclusions.gen.py | 10 +-- .../mandatory_inclusions.gen.py | 3 +- libcxx/utils/libcxx/test/format.py | 24 ++++-- 10 files changed, 97 insertions(+), 105 deletions(-) create mode 100644 libcxx/test/selftest/gen.cpp/unsupported.gen.cpp diff --git a/libcxx/test/extensions/clang/clang_modules_include.gen.py b/libcxx/test/extensions/clang/clang_modules_include.gen.py index 379ac22c8f47f..562ce3b94d10b 100644 --- a/libcxx/test/extensions/clang/clang_modules_include.gen.py +++ b/libcxx/test/extensions/clang/clang_modules_include.gen.py @@ -10,9 +10,32 @@ # This is important notably because the LLDB data formatters use # libc++ headers with modules enabled. -# RUN: %{python} %s %{libcxx-dir}/utils +# Older macOS SDKs were not properly modularized, which causes issues with localization. +# This feature should instead be based on the SDK version. +# UNSUPPORTED: stdlib=system && target={{.+}}-apple-macosx13{{.*}} + +# GCC doesn't support -fcxx-modules +# UNSUPPORTED: gcc + +# The Windows headers don't appear to be compatible with modules +# UNSUPPORTED: windows +# UNSUPPORTED: buildhost=windows + +# The Android headers don't appear to be compatible with modules yet +# UNSUPPORTED: LIBCXX-ANDROID-FIXME + +# TODO: Investigate this failure +# UNSUPPORTED: LIBCXX-FREEBSD-FIXME + +# TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored +# UNSUPPORTED: LIBCXX-PICOLIBC-FIXME + +# TODO: Fix seemingly circular inclusion or on AIX +# UNSUPPORTED: LIBCXX-AIX-FIXME -# block Lit from interpreting a RUN/XFAIL/etc inside the generation script +# UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME + +# RUN: %{python} %s %{libcxx-dir}/utils # END. import sys @@ -29,31 +52,6 @@ //--- {header}.compile.pass.cpp // RUN: %{{cxx}} %s %{{flags}} %{{compile_flags}} -fmodules -fcxx-modules -fmodules-cache-path=%t -fsyntax-only -// Older macOS SDKs were not properly modularized, which causes issues with localization. -// This feature should instead be based on the SDK version. -// UNSUPPORTED: stdlib=system && target={{{{.+}}}}-apple-macosx13{{{{.*}}}} - -// GCC doesn't support -fcxx-modules -// UNSUPPORTED: gcc - -// The Windows headers don't appear to be compatible with modules -// UNSUPPORTED: windows -// UNSUPPORTED: buildhost=windows - -// The Android headers don't appear to be compatible with modules yet -// UNSUPPORTED: LIBCXX-ANDROID-FIXME - -// TODO: Investigate this failure -// UNSUPPORTED: LIBCXX-FREEBSD-FIXME - -// TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored -// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME - -// TODO: Fix seemingly circular inclusion or on AIX -// UNSUPPORTED: LIBCXX-AIX-FIXME - -// UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME - {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} @@ -66,32 +64,6 @@ //--- import_std.compile.pass.mm // RUN: %{{cxx}} %s %{{flags}} %{{compile_flags}} -fmodules -fcxx-modules -fmodules-cache-path=%t -fsyntax-only -// REQUIRES: clang-modules-build - -// Older macOS SDKs were not properly modularized, which causes issues with localization. -// This feature should instead be based on the SDK version. -// UNSUPPORTED: stdlib=system && target={{{{.+}}}}-apple-macosx13{{{{.*}}}} - -// GCC doesn't support -fcxx-modules -// UNSUPPORTED: gcc - -// The Windows headers don't appear to be compatible with modules -// UNSUPPORTED: windows -// UNSUPPORTED: buildhost=windows - -// The Android headers don't appear to be compatible with modules yet -// UNSUPPORTED: LIBCXX-ANDROID-FIXME - -// TODO: Investigate this failure -// UNSUPPORTED: LIBCXX-FREEBSD-FIXME - -// TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored -// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME - -// TODO: Fix seemingly circular inclusion or on AIX -// UNSUPPORTED: LIBCXX-AIX-FIXME - @import std; - """ ) diff --git a/libcxx/test/extensions/posix/xopen_source.gen.py b/libcxx/test/extensions/posix/xopen_source.gen.py index d4a3651181ca7..cfdb1c9f656ef 100644 --- a/libcxx/test/extensions/posix/xopen_source.gen.py +++ b/libcxx/test/extensions/posix/xopen_source.gen.py @@ -12,6 +12,18 @@ # # https://github.com/llvm/llvm-project/issues/117630 +# Some parts of the code like use non-standard functions in their implementation, +# and these functions are not provided when _XOPEN_SOURCE is set to older values. This +# breaks when building with modules even when we don't use the offending headers directly. +# UNSUPPORTED: clang-modules-build + +# The AIX localization support uses some functions as part of their headers that require a +# recent value of _XOPEN_SOURCE. +# UNSUPPORTED: LIBCXX-AIX-FIXME + +# This test fails on FreeBSD for an unknown reason. +# UNSUPPORTED: LIBCXX-FREEBSD-FIXME + # RUN: %{python} %s %{libcxx-dir}/utils # END. @@ -33,19 +45,6 @@ print( f"""\ //--- {header}.xopen_source_{version}.compile.pass.cpp - -// Some parts of the code like use non-standard functions in their implementation, -// and these functions are not provided when _XOPEN_SOURCE is set to older values. This -// breaks when building with modules even when we don't use the offending headers directly. -// UNSUPPORTED: clang-modules-build - -// The AIX localization support uses some functions as part of their headers that require a -// recent value of _XOPEN_SOURCE. -// UNSUPPORTED: LIBCXX-AIX-FIXME - -// This test fails on FreeBSD for an unknown reason. -// UNSUPPORTED: LIBCXX-FREEBSD-FIXME - {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} diff --git a/libcxx/test/libcxx-03/clang_tidy.gen.py b/libcxx/test/libcxx-03/clang_tidy.gen.py index dbab2875e3126..5926f521e780e 100644 --- a/libcxx/test/libcxx-03/clang_tidy.gen.py +++ b/libcxx/test/libcxx-03/clang_tidy.gen.py @@ -6,12 +6,17 @@ # # ===----------------------------------------------------------------------===## - # Run our custom libc++ clang-tidy checks on all public headers. -# RUN: %{python} %s %{libcxx-dir}/utils +# REQUIRES: has-clang-tidy + +# The frozen headers should not be updated to the latest libc++ style, so don't test. +# UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME -# block Lit from interpreting a RUN/XFAIL/etc inside the generation script +# The GCC compiler flags are not always compatible with clang-tidy. +# UNSUPPORTED: gcc + +# RUN: %{python} %s %{libcxx-dir}/utils # END. import sys @@ -21,15 +26,6 @@ for header in public_headers: print(f"""\ //--- {header}.sh.cpp - -// REQUIRES: has-clang-tidy - -// The frozen headers should not be updated to the latest libc++ style, so don't test. -// UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME - -// The GCC compiler flags are not always compatible with clang-tidy. -// UNSUPPORTED: gcc - {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} diff --git a/libcxx/test/libcxx/clang_tidy.gen.py b/libcxx/test/libcxx/clang_tidy.gen.py index 1663f5c396bbf..16c90c3ef7130 100644 --- a/libcxx/test/libcxx/clang_tidy.gen.py +++ b/libcxx/test/libcxx/clang_tidy.gen.py @@ -6,12 +6,14 @@ # # ===----------------------------------------------------------------------===## - # Run our custom libc++ clang-tidy checks on all public headers. -# RUN: %{python} %s %{libcxx-dir}/utils +# REQUIRES: has-clang-tidy -# block Lit from interpreting a RUN/XFAIL/etc inside the generation script +# The GCC compiler flags are not always compatible with clang-tidy. +# UNSUPPORTED: gcc + +# RUN: %{python} %s %{libcxx-dir}/utils # END. import sys @@ -21,12 +23,6 @@ for header in public_headers: print(f"""\ //--- {header}.sh.cpp - -// REQUIRES: has-clang-tidy - -// The GCC compiler flags are not always compatible with clang-tidy. -// UNSUPPORTED: gcc - {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} diff --git a/libcxx/test/libcxx/transitive_includes.gen.py b/libcxx/test/libcxx/transitive_includes.gen.py index f01dbac26a8e8..6ed35af7e275e 100644 --- a/libcxx/test/libcxx/transitive_includes.gen.py +++ b/libcxx/test/libcxx/transitive_includes.gen.py @@ -17,8 +17,6 @@ # to avoid breaking users at every release. # RUN: %{python} %s %{libcxx-dir}/utils - -# block Lit from interpreting a RUN/XFAIL/etc inside the generation script # END. import sys diff --git a/libcxx/test/selftest/gen.cpp/unsupported.gen.cpp b/libcxx/test/selftest/gen.cpp/unsupported.gen.cpp new file mode 100644 index 0000000000000..6f35939f22582 --- /dev/null +++ b/libcxx/test/selftest/gen.cpp/unsupported.gen.cpp @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// Make sure we can mark a gen-test as UNSUPPORTED + +// We use C++03 as a random feature that we know exists. The goal is to make +// this test always unsupported. +// UNSUPPORTED: c++03 +// REQUIRES: c++03 + +// Note that an unsupported gen-test should still contain some commands, otherwise +// what are we generating? They are never executed, though. +// RUN: something-definitely-invalid diff --git a/libcxx/test/std/double_include.gen.py b/libcxx/test/std/double_include.gen.py index fcf3b9a8fa2e0..f019710be55b2 100644 --- a/libcxx/test/std/double_include.gen.py +++ b/libcxx/test/std/double_include.gen.py @@ -8,9 +8,10 @@ # Test that we can include each header in two TU's and link them together. -# RUN: %{python} %s %{libcxx-dir}/utils +# We're using compiler-specific flags in this test +# REQUIRES: (gcc || clang) -# Block Lit from interpreting a RUN/XFAIL/etc inside the generation script. +# RUN: %{python} %s %{libcxx-dir}/utils # END. import sys @@ -28,9 +29,6 @@ {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} -// We're using compiler-specific flags in this test -// REQUIRES: (gcc || clang) - // RUN: %{{cxx}} -c %s -o %t.first.o %{{flags}} %{{compile_flags}} // RUN: %{{cxx}} -c %s -o %t.second.o -DWITH_MAIN %{{flags}} %{{compile_flags}} // RUN: %{{cxx}} -o %t.exe %t.first.o %t.second.o %{{flags}} %{{link_flags}} diff --git a/libcxx/test/std/header_inclusions.gen.py b/libcxx/test/std/header_inclusions.gen.py index 8ff93810069fa..cebff94fdd1ab 100644 --- a/libcxx/test/std/header_inclusions.gen.py +++ b/libcxx/test/std/header_inclusions.gen.py @@ -9,6 +9,11 @@ # Test that all headers include all the other headers they're supposed to, as # prescribed by the Standard. +# UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME + +# TODO: This is currently a libc++-specific way of testing the includes, but is a requirement for all implementation +# REQUIRES: stdlib=libc++ + # RUN: %{python} %s %{libcxx-dir}/utils # END. @@ -46,11 +51,6 @@ print( f"""\ //--- {header}.compile.pass.cpp -// UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME - -// TODO: This is currently a libc++-specific way of testing the includes, but is a requirement for all implementation -// REQUIRES: stdlib=libc++ - {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} diff --git a/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py b/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py index 46290b1ed52a0..34ede107e5ed6 100644 --- a/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py +++ b/libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py @@ -12,6 +12,8 @@ # , , , , , , , , # , . +# UNSUPPORTED: c++03 + # RUN: %{python} %s %{libcxx-dir}/utils # END. @@ -53,7 +55,6 @@ //--- {header}.pass.cpp {lit_header_restrictions.get(header, '')} {lit_header_undeprecations.get(header, '')} -// UNSUPPORTED: c++03 #include <{header}> #include diff --git a/libcxx/utils/libcxx/test/format.py b/libcxx/utils/libcxx/test/format.py index 59d0fffd37819..a0b7b5bdb5b9f 100644 --- a/libcxx/utils/libcxx/test/format.py +++ b/libcxx/utils/libcxx/test/format.py @@ -34,11 +34,14 @@ def _checkBaseSubstitutions(substitutions): def _executeScriptInternal(test, litConfig, commands): """ - Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands) + Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands), or an appropriate lit.Test.Result + in case of an error while parsing the script. TODO: This really should be easier to access from Lit itself """ parsedCommands = parseScript(test, preamble=commands) + if isinstance(parsedCommands, lit.Test.Result): + return parsedCommands _, tmpBase = _getTempPaths(test) execDir = os.path.dirname(test.getExecPath()) @@ -65,7 +68,8 @@ def parseScript(test, preamble): """ Extract the script from a test, with substitutions applied. - Returns a list of commands ready to be executed. + Returns a list of commands ready to be executed, or an appropriate lit.Test.Result in case of error + while parsing the script (this includes the script being unsupported). - test The lit.Test to parse. @@ -350,6 +354,8 @@ def execute(self, test, litConfig): if "enable-benchmarks=run" in test.config.available_features: steps += ["%dbg(EXECUTED AS) %{exec} %t.exe --benchmark_out=%T/benchmark-result.json --benchmark_out_format=json"] return self._executeShTest(test, litConfig, steps) + elif re.search('[.]gen[.][^.]+$', filename): # This only happens when a generator test is not supported + return self._executeShTest(test, litConfig, []) else: return lit.Test.Result( lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename) @@ -381,11 +387,19 @@ def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig): generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite)) os.makedirs(generatorExecDir, exist_ok=True) - # Run the generator test + # Run the generator test. It's possible for this to fail for two reasons: the generator test + # is unsupported or the generator ran but failed at runtime -- handle both. In the first case, + # we return the generator test itself, since it should produce the same result when run after + # test suite generation. In the second case, it's a true error so we report it. steps = [] # Steps must already be in the script - (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps) + result = _executeScriptInternal(generator, litConfig, steps) + if isinstance(result, lit.Test.Result): + yield generator + return + + (out, err, exitCode, _, _) = result if exitCode != 0: - raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}") + raise RuntimeError(f"Error while trying to generate gen test {'/'.join(pathInSuite)}\nstdout:\n{out}\n\nstderr:\n{err}") # Split the generated output into multiple files and generate one test for each file for subfile, content in self._splitFile(out): From 893ad2134551b94336c476d62c1a276b4c1b1a98 Mon Sep 17 00:00:00 2001 From: Louis Dionne Date: Wed, 3 Sep 2025 15:54:26 -0400 Subject: [PATCH 2/2] Leave alone some REQUIRES --- libcxx/test/extensions/clang/clang_modules_include.gen.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libcxx/test/extensions/clang/clang_modules_include.gen.py b/libcxx/test/extensions/clang/clang_modules_include.gen.py index 562ce3b94d10b..28661049d6e85 100644 --- a/libcxx/test/extensions/clang/clang_modules_include.gen.py +++ b/libcxx/test/extensions/clang/clang_modules_include.gen.py @@ -64,6 +64,8 @@ //--- import_std.compile.pass.mm // RUN: %{{cxx}} %s %{{flags}} %{{compile_flags}} -fmodules -fcxx-modules -fmodules-cache-path=%t -fsyntax-only +// REQUIRES: clang-modules-build + @import std; """ )