[libc++] Add support for generated tests in the libc++ test format

A recurring problem recently has been that libc++ has several generated
tests which all need to be re-generated before committing a change. This
creates noise during code reviews and friction for contributors.

Furthermore, the way we generated most of these tests resulted in
extremely bad compilation times when using modules, because we defined
a macro before compiling each file.

This commit introduces a new kind of test called a '.gen' test. These
tests are normal shell tests, however the Lit test format will run the
test to discover the actual Lit tests it should run. This basically
allows generating a Lit test suite on the fly using arbitrary code,
which can be used in the future to generate tests like our __verbose_abort
tests and several others.

Differential Revision: https://reviews.llvm.org/D151258

Author: ldionne

libcxx/test/libcxx/selftest/gen.cpp/empty.gen.cpp

@@ -0,0 +1,11 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate no tests at all
+
+// RUN: true

libcxx/test/libcxx/selftest/gen.cpp/one.gen.cpp

@@ -0,0 +1,11 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate one test
+
+// RUN: echo "//--- test1.compile.pass.cpp"

libcxx/test/libcxx/selftest/gen.cpp/two.gen.cpp

@@ -0,0 +1,12 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// Make sure we can generate two tests
+
+// RUN: echo "//--- test1.compile.pass.cpp"
+// RUN: echo "//--- test2.compile.pass.cpp"

libcxx/utils/libcxx/test/dsl.py

@@ -179,9 +179,7 @@ def programOutput(config, program, args=None):
                 "Failed to run program, cmd:\n{}\nstderr is:\n{}".format(runcmd, err)
             )
 
-        actualOut = re.search("# command output:\n(.+)\n$", out, flags=re.DOTALL)
-        actualOut = actualOut.group(1) if actualOut else ""
-        return actualOut
+        return libcxx.test.format._parseLitOutput(out)
 
 
 @_memoizeExpensiveOperation(

libcxx/utils/libcxx/test/format.py

@@ -6,6 +6,8 @@
 #
 # ===----------------------------------------------------------------------===##
 
+import contextlib
+import io
 import lit
 import lit.formats
 import os
@@ -33,6 +35,38 @@ def _checkBaseSubstitutions(substitutions):
     for s in ["%{cxx}", "%{compile_flags}", "%{link_flags}", "%{flags}", "%{exec}"]:
         assert s in substitutions, "Required substitution {} was not provided".format(s)
 
+def _parseLitOutput(fullOutput):
+    """
+    Parse output of a Lit ShTest to extract the actual output of the contained commands.
+
+    This takes output of the form
+
+        $ ":" "RUN: at line 11"
+        $ "echo" "OUTPUT1"
+        # command output:
+        OUTPUT1
+
+        $ ":" "RUN: at line 12"
+        $ "echo" "OUTPUT2"
+        # command output:
+        OUTPUT2
+
+    and returns a string containing
+
+        OUTPUT1
+        OUTPUT2
+
+    as-if the commands had been run directly. This is a workaround for the fact
+    that Lit doesn't let us execute ShTest and retrieve the raw output without
+    injecting additional Lit output around it.
+    """
+    parsed = ''
+    for output in re.split('[$]\s*":"\s*"RUN: at line \d+"', fullOutput):
+        if output: # skip blank lines
+            commandOutput = re.search("# command output:\n(.+)\n$", output, flags=re.DOTALL)
+            if commandOutput:
+                parsed += commandOutput.group(1)
+    return parsed
 
 def _executeScriptInternal(test, litConfig, commands):
     """
@@ -170,6 +204,16 @@ class CxxStandardLibraryTest(lit.formats.TestFormat):
 
     FOO.sh.<anything>       - A builtin Lit Shell test
 
+    FOO.gen.<anything>      - A .sh test that generates one or more Lit tests on the
+                              fly. Executing this test must generate one or more files
+                              as expected by LLVM split-file, and each generated file
+                              leads to a separate Lit test that runs that file as
+                              defined by the test format. This can be used to generate
+                              multiple Lit tests from a single source file, which is
+                              useful for testing repetitive properties in the library.
+                              Be careful not to abuse this since this is not a replacement
+                              for usual code reuse techniques.
+
     FOO.verify.cpp          - Compiles with clang-verify. This type of test is
                               automatically marked as UNSUPPORTED if the compiler
                               does not support Clang-verify.
@@ -245,6 +289,7 @@ def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
             "[.]link[.]pass[.]mm$",
             "[.]link[.]fail[.]cpp$",
             "[.]sh[.][^.]+$",
+            "[.]gen[.][^.]+$",
             "[.]verify[.]cpp$",
             "[.]fail[.]cpp$",
         ]
@@ -257,9 +302,13 @@ def getTestsInDirectory(self, testSuite, pathInSuite, litConfig, localConfig):
             filepath = os.path.join(sourcePath, filename)
             if not os.path.isdir(filepath):
                 if any([re.search(ext, filename) for ext in SUPPORTED_SUFFIXES]):
-                    yield lit.Test.Test(
-                        testSuite, pathInSuite + (filename,), localConfig
-                    )
+                    # If this is a generated test, run the generation step and add
+                    # as many Lit tests as necessary.
+                    if re.search('[.]gen[.][^.]+$', filename):
+                        for test in self._generateGenTest(testSuite, pathInSuite + (filename,), litConfig, localConfig):
+                            yield test
+                    else:
+                        yield lit.Test.Test(testSuite, pathInSuite + (filename,), localConfig)
 
     def execute(self, test, litConfig):
         VERIFY_FLAGS = (
@@ -356,3 +405,42 @@ def _executeShTest(self, test, litConfig, steps):
             return lit.TestRunner._runShTest(
                 test, litConfig, useExternalSh, script, tmpBase
             )
+
+    def _generateGenTest(self, testSuite, pathInSuite, litConfig, localConfig):
+        generator = lit.Test.Test(testSuite, pathInSuite, localConfig)
+
+        # Make sure we have a directory to execute the generator test in
+        generatorExecDir = os.path.dirname(testSuite.getExecPath(pathInSuite))
+        os.makedirs(generatorExecDir, exist_ok=True)
+
+        # Run the generator test
+        steps = [] # Steps must already be in the script
+        (out, err, exitCode, _, _) = _executeScriptInternal(generator, litConfig, steps)
+        if exitCode != 0:
+            raise RuntimeError(f"Error while trying to generate gen test\nstdout:\n{out}\n\nstderr:\n{err}")
+
+        # Split the generated output into multiple files and generate one test for each file
+        parsed = _parseLitOutput(out)
+        for (subfile, content) in self._splitFile(parsed):
+            generatedFile = testSuite.getExecPath(pathInSuite + (subfile, ))
+            os.makedirs(os.path.dirname(generatedFile), exist_ok=True)
+            with open(generatedFile, 'w') as f:
+                f.write(content)
+            yield lit.Test.Test(testSuite, (generatedFile,), localConfig)
+
+    def _splitFile(self, input):
+        DELIM = r'^(//|#)---(.+)'
+        lines = input.splitlines()
+        currentFile = None
+        thisFileContent = []
+        for line in lines:
+            match = re.match(DELIM, line)
+            if match:
+                if currentFile is not None:
+                    yield (currentFile, '\n'.join(thisFileContent))
+                currentFile = match.group(2).strip()
+                thisFileContent = []
+            assert currentFile is not None, f"Some input to split-file doesn't belong to any file, input was:\n{input}"
+            thisFileContent.append(line)
+        if currentFile is not None:
+            yield (currentFile, '\n'.join(thisFileContent))