[libc++] Make it possible to mark a gen-test as UNSUPPORTED

libcxx/test/extensions/clang/clang_modules_include.gen.py

@@ -10,9 +10,32 @@
 # This is important notably because the LLDB data formatters use
 # libc++ headers with modules enabled.
 
-# RUN: %{python} %s %{libcxx-dir}/utils
+# Older macOS SDKs were not properly modularized, which causes issues with localization.
+# This feature should instead be based on the SDK version.
+# UNSUPPORTED: stdlib=system && target={{.+}}-apple-macosx13{{.*}}
+
+# GCC doesn't support -fcxx-modules
+# UNSUPPORTED: gcc
+
+# The Windows headers don't appear to be compatible with modules
+# UNSUPPORTED: windows
+# UNSUPPORTED: buildhost=windows
+
+# The Android headers don't appear to be compatible with modules yet
+# UNSUPPORTED: LIBCXX-ANDROID-FIXME
+
+# TODO: Investigate this failure
+# UNSUPPORTED: LIBCXX-FREEBSD-FIXME
+
+# TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored
+# UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
+
+# TODO: Fix seemingly circular inclusion or <wchar.h> on AIX
+# UNSUPPORTED: LIBCXX-AIX-FIXME
 
-# block Lit from interpreting a RUN/XFAIL/etc inside the generation script
+# UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME
+
+# RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
 import sys
@@ -29,31 +52,6 @@
 //--- {header}.compile.pass.cpp
 // RUN: %{{cxx}} %s %{{flags}} %{{compile_flags}} -fmodules -fcxx-modules -fmodules-cache-path=%t -fsyntax-only
 
-// Older macOS SDKs were not properly modularized, which causes issues with localization.
-// This feature should instead be based on the SDK version.
-// UNSUPPORTED: stdlib=system && target={{{{.+}}}}-apple-macosx13{{{{.*}}}}
-
-// GCC doesn't support -fcxx-modules
-// UNSUPPORTED: gcc
-
-// The Windows headers don't appear to be compatible with modules
-// UNSUPPORTED: windows
-// UNSUPPORTED: buildhost=windows
-
-// The Android headers don't appear to be compatible with modules yet
-// UNSUPPORTED: LIBCXX-ANDROID-FIXME
-
-// TODO: Investigate this failure
-// UNSUPPORTED: LIBCXX-FREEBSD-FIXME
-
-// TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored
-// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
-
-// TODO: Fix seemingly circular inclusion or <wchar.h> on AIX
-// UNSUPPORTED: LIBCXX-AIX-FIXME
-
-// UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME
-
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 
@@ -68,30 +66,6 @@
 
 // REQUIRES: clang-modules-build
 
-// Older macOS SDKs were not properly modularized, which causes issues with localization.
-// This feature should instead be based on the SDK version.
-// UNSUPPORTED: stdlib=system && target={{{{.+}}}}-apple-macosx13{{{{.*}}}}
-
-// GCC doesn't support -fcxx-modules
-// UNSUPPORTED: gcc
-
-// The Windows headers don't appear to be compatible with modules
-// UNSUPPORTED: windows
-// UNSUPPORTED: buildhost=windows
-
-// The Android headers don't appear to be compatible with modules yet
-// UNSUPPORTED: LIBCXX-ANDROID-FIXME
-
-// TODO: Investigate this failure
-// UNSUPPORTED: LIBCXX-FREEBSD-FIXME
-
-// TODO: Investigate why this doesn't work on Picolibc once the locale base API is refactored
-// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME
-
-// TODO: Fix seemingly circular inclusion or <wchar.h> on AIX
-// UNSUPPORTED: LIBCXX-AIX-FIXME
-
 @import std;
-
 """
 )

libcxx/test/extensions/posix/xopen_source.gen.py

@@ -12,6 +12,18 @@
 #
 # https://llvm.org/PR117630
 
+# Some parts of the code like <fstream> use non-standard functions in their implementation,
+# and these functions are not provided when _XOPEN_SOURCE is set to older values. This
+# breaks when building with modules even when we don't use the offending headers directly.
+# UNSUPPORTED: clang-modules-build
+
+# The AIX localization support uses some functions as part of their headers that require a
+# recent value of _XOPEN_SOURCE.
+# UNSUPPORTED: LIBCXX-AIX-FIXME
+
+# This test fails on FreeBSD for an unknown reason.
+# UNSUPPORTED: LIBCXX-FREEBSD-FIXME
+
 # RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
@@ -33,19 +45,6 @@
         print(
             f"""\
 //--- {header}.xopen_source_{version}.compile.pass.cpp
-
-// Some parts of the code like <fstream> use non-standard functions in their implementation,
-// and these functions are not provided when _XOPEN_SOURCE is set to older values. This
-// breaks when building with modules even when we don't use the offending headers directly.
-// UNSUPPORTED: clang-modules-build
-
-// The AIX localization support uses some functions as part of their headers that require a
-// recent value of _XOPEN_SOURCE.
-// UNSUPPORTED: LIBCXX-AIX-FIXME
-
-// This test fails on FreeBSD for an unknown reason.
-// UNSUPPORTED: LIBCXX-FREEBSD-FIXME
-
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 

libcxx/test/libcxx-03/clang_tidy.gen.py

@@ -6,12 +6,17 @@
 #
 # ===----------------------------------------------------------------------===##
 
-
 # Run our custom libc++ clang-tidy checks on all public headers.
 
-# RUN: %{python} %s %{libcxx-dir}/utils
+# REQUIRES: has-clang-tidy
+
+# The frozen headers should not be updated to the latest libc++ style, so don't test.
+# UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME
 
-# block Lit from interpreting a RUN/XFAIL/etc inside the generation script
+# The GCC compiler flags are not always compatible with clang-tidy.
+# UNSUPPORTED: gcc
+
+# RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
 import sys
@@ -21,15 +26,6 @@
 for header in public_headers:
   print(f"""\
 //--- {header}.sh.cpp
-
-// REQUIRES: has-clang-tidy
-
-// The frozen headers should not be updated to the latest libc++ style, so don't test.
-// UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME
-
-// The GCC compiler flags are not always compatible with clang-tidy.
-// UNSUPPORTED: gcc
-
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 

libcxx/test/libcxx/clang_tidy.gen.py

@@ -6,12 +6,14 @@
 #
 # ===----------------------------------------------------------------------===##
 
-
 # Run our custom libc++ clang-tidy checks on all public headers.
 
-# RUN: %{python} %s %{libcxx-dir}/utils
+# REQUIRES: has-clang-tidy
 
-# block Lit from interpreting a RUN/XFAIL/etc inside the generation script
+# The GCC compiler flags are not always compatible with clang-tidy.
+# UNSUPPORTED: gcc
+
+# RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
 import sys
@@ -21,12 +23,6 @@
 for header in public_headers:
   print(f"""\
 //--- {header}.sh.cpp
-
-// REQUIRES: has-clang-tidy
-
-// The GCC compiler flags are not always compatible with clang-tidy.
-// UNSUPPORTED: gcc
-
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 

libcxx/test/libcxx/transitive_includes.gen.py

@@ -17,8 +17,6 @@
 # to avoid breaking users at every release.
 
 # RUN: %{python} %s %{libcxx-dir}/utils
-
-# block Lit from interpreting a RUN/XFAIL/etc inside the generation script
 # END.
 
 import sys

libcxx/test/selftest/gen.cpp/unsupported.gen.cpp

@@ -0,0 +1,18 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can mark a gen-test as UNSUPPORTED
+
+// We use C++03 as a random feature that we know exists. The goal is to make
+// this test always unsupported.
+// UNSUPPORTED: c++03
+// REQUIRES: c++03
+
+// Note that an unsupported gen-test should still contain some commands, otherwise
+// what are we generating? They are never executed, though.
+// RUN: something-definitely-invalid

libcxx/test/std/double_include.gen.py

@@ -8,9 +8,10 @@
 
 # Test that we can include each header in two TU's and link them together.
 
-# RUN: %{python} %s %{libcxx-dir}/utils
+# We're using compiler-specific flags in this test
+# REQUIRES: (gcc || clang)
 
-# Block Lit from interpreting a RUN/XFAIL/etc inside the generation script.
+# RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
 import sys
@@ -28,9 +29,6 @@
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 
-// We're using compiler-specific flags in this test
-// REQUIRES: (gcc || clang)
-
 // RUN: %{{cxx}} -c %s -o %t.first.o %{{flags}} %{{compile_flags}}
 // RUN: %{{cxx}} -c %s -o %t.second.o -DWITH_MAIN %{{flags}} %{{compile_flags}}
 // RUN: %{{cxx}} -o %t.exe %t.first.o %t.second.o %{{flags}} %{{link_flags}}

libcxx/test/std/header_inclusions.gen.py

@@ -9,6 +9,11 @@
 # Test that all headers include all the other headers they're supposed to, as
 # prescribed by the Standard.
 
+# UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME
+
+# TODO: This is currently a libc++-specific way of testing the includes, but is a requirement for all implementation
+# REQUIRES: stdlib=libc++
+
 # RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
@@ -46,11 +51,6 @@
     print(
         f"""\
 //--- {header}.compile.pass.cpp
-// UNSUPPORTED: FROZEN-CXX03-HEADERS-FIXME
-
-// TODO: This is currently a libc++-specific way of testing the includes, but is a requirement for all implementation
-// REQUIRES: stdlib=libc++
-
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
 

libcxx/test/std/iterators/iterator.range/mandatory_inclusions.gen.py

@@ -12,6 +12,8 @@
 # <list>, <map>, <regex>, <set>, <span>, <string>, <string_view>, <unordered_map>,
 # <unordered_set>, <vector>.
 
+# UNSUPPORTED: c++03
+
 # RUN: %{python} %s %{libcxx-dir}/utils
 # END.
 
@@ -53,7 +55,6 @@
 //--- {header}.pass.cpp
 {lit_header_restrictions.get(header, '')}
 {lit_header_undeprecations.get(header, '')}
-// UNSUPPORTED: c++03
 
 #include <{header}>
 #include <cassert>

libcxx/utils/libcxx/test/format.py

@@ -34,11 +34,14 @@ def _checkBaseSubstitutions(substitutions):
 
 def _executeScriptInternal(test, litConfig, commands):
     """
-    Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands)
+    Returns (stdout, stderr, exitCode, timeoutInfo, parsedCommands), or an appropriate lit.Test.Result
+    in case of an error while parsing the script.
 
     TODO: This really should be easier to access from Lit itself
     """
     parsedCommands = parseScript(test, preamble=commands)
+    if isinstance(parsedCommands, lit.Test.Result):
+        return parsedCommands
 
     _, tmpBase = _getTempPaths(test)
     execDir = os.path.dirname(test.getExecPath())
@@ -65,7 +68,8 @@ def parseScript(test, preamble):
     """
     Extract the script from a test, with substitutions applied.
 
-    Returns a list of commands ready to be executed.
+    Returns a list of commands ready to be executed, or an appropriate lit.Test.Result in case of error
+    while parsing the script (this includes the script being unsupported).
 
     - test
         The lit.Test to parse.
@@ -350,6 +354,8 @@ def execute(self, test, litConfig):
             if "enable-benchmarks=run" in test.config.available_features:
                 steps += ["%dbg(EXECUTED AS) %{exec} %t.exe --benchmark_out=%T/benchmark-result.json --benchmark_out_format=json"]
             return self._executeShTest(test, litConfig, steps)
+        elif re.search('[.]gen[.][^.]+$', filename): # This only happens when a generator test is not supported
+            return self._executeShTest(test, litConfig, [])
         else:
             return lit.Test.Result(
                 lit.Test.UNRESOLVED, "Unknown test suffix for '{}'".format(filename)
@@ -381,11 +387,19 @@ def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
         generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
         os.makedirs(generatorExecDir, exist_ok=True)
 
-        # Run the generator test
+        # Run the generator test. It's possible for this to fail for two reasons: the generator test
+        # is unsupported or the generator ran but failed at runtime -- handle both. In the first case,
+        # we return the generator test itself, since it should produce the same result when run after
+        # test suite generation. In the second case, it's a true error so we report it.
         steps = [] # Steps must already be in the script
-        (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
+        result = _executeScriptInternal(generator, litConfig, steps)
+        if isinstance(result, lit.Test.Result):
+            yield generator
+            return
+
+        (out, err, exitCode, _, _) = result
         if exitCode != 0:
-            raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}")
+            raise RuntimeError(f"Error while trying to generate gen test {'/'.join(pathInSuite)}\nstdout:\n{out}\n\nstderr:\n{err}")
 
         # Split the generated output into multiple files and generate one test for each file
         for subfile, content in self._splitFile(out):