Add lit config for dir with standalone tests

Some test systems do not use lit for test discovery but only for its
substitution and test selection because they use another way of managing
test collections, e.g. CTest. This forces those tests to be invoked with
lit --no-indirectly-run-check. When a mix of lit version is in use, it
requires to detect the availability of that option.

This commit provides a new config option standalone_tests to signal a
directory made of tests meant to run as standalone. When this option is
set, lit skips test discovery and the indirectly run check. It also adds
the missing documentation for --no-indirectly-run-check.

Reviewed By: jdenny

Differential Revision: https://reviews.llvm.org/D94766

llvm/docs/CommandGuide/lit.rst

@@ -154,6 +154,11 @@ EXECUTION OPTIONS
  suite take the most time to execute.  Note that this option is most useful
  with ``-j 1``.
 
+.. option:: --no-indirectly-run-check
+
+ Do not error if a test would not be run if the user had specified the
+ containing directory instead of naming the test directly.
+
 .. _selection-options:
 
 SELECTION OPTIONS
@@ -372,6 +377,11 @@ executed, two important global variables are predefined:
  **environment** A dictionary representing the environment to use when executing
  tests in the suite.
 
+ **standalone_tests** When true, mark a directory with tests expected to be run
+ standalone. Test discovery is disabled for that directory and
+ *--no-indirectly-run-check* is in effect. *lit.suffixes* and *lit.excludes*
+ must be empty when this variable is true.
+
  **suffixes** For **lit** test formats which scan directories for tests, this
  variable is a list of suffixes to identify test files.  Used by: *ShTest*.
 

llvm/utils/lit/lit/TestingConfig.py

@@ -60,7 +60,8 @@ def fromdefaults(litConfig):
                              test_source_root = None,
                              excludes = [],
                              available_features = available_features,
-                             pipefail = True)
+                             pipefail = True,
+                             standalone_tests = False)
 
     def load_from_path(self, path, litConfig):
         """
@@ -105,7 +106,8 @@ def __init__(self, parent, name, suffixes, test_format,
                  environment, substitutions, unsupported,
                  test_exec_root, test_source_root, excludes,
                  available_features, pipefail, limit_to_features = [],
-                 is_early = False, parallelism_group = None):
+                 is_early = False, parallelism_group = None,
+                 standalone_tests = False):
         self.parent = parent
         self.name = str(name)
         self.suffixes = set(suffixes)
@@ -118,6 +120,7 @@ def __init__(self, parent, name, suffixes, test_format,
         self.excludes = set(excludes)
         self.available_features = set(available_features)
         self.pipefail = pipefail
+        self.standalone_tests = standalone_tests
         # This list is used by TestRunner.py to restrict running only tests that
         # require one of the features in this list if this list is non-empty.
         # Configurations can set this list to restrict the set of tests to run.

llvm/utils/lit/lit/discovery.py

@@ -160,8 +160,13 @@ def getTestsInSuite(ts, path_in_suite, litConfig,
         # tests which are not executed. The check adds some performance
         # overhead which might be important if a large number of tests
         # are being run directly.
-        # --no-indirectly-run-check: skips this check.
-        if indirectlyRunCheck and lc.test_format is not None:
+        # This check can be disabled by using --no-indirectly-run-check or
+        # setting the standalone_tests variable in the suite's configuration.
+        if (
+            indirectlyRunCheck
+            and lc.test_format is not None
+            and not lc.standalone_tests
+        ):
             found = False
             for res in lc.test_format.getTestsInDirectory(ts, test_dir_in_suite,
                                                           litConfig, lc):
@@ -171,6 +176,7 @@ def getTestsInSuite(ts, path_in_suite, litConfig,
             if not found:
                 litConfig.error(
                     '%r would not be run indirectly: change name or LIT config'
+                    '(e.g. suffixes or standalone_tests variables)'
                     % test.getFullName())
 
         yield test
@@ -180,6 +186,15 @@ def getTestsInSuite(ts, path_in_suite, litConfig,
     # local configuration.
     lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache)
 
+    # Directory contains tests to be run standalone. Do not try to discover.
+    if lc.standalone_tests:
+        if lc.suffixes or lc.excludes:
+            litConfig.warning(
+                'standalone_tests set in LIT config but suffixes or excludes'
+                    ' are also set'
+            )
+        return
+
     # Search for tests.
     if lc.test_format is not None:
         for res in lc.test_format.getTestsInDirectory(ts, path_in_suite,

llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/lit.cfg

@@ -0,0 +1,5 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.excludes = ['.test']
+config.standalone_tests = True

llvm/utils/lit/tests/Inputs/standalone-tests-with-excludes/true.txt

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/lit.cfg

@@ -0,0 +1,5 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.suffixes = ['.txt']
+config.standalone_tests = True

llvm/utils/lit/tests/Inputs/standalone-tests-with-suffixes/true.txt

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/Inputs/standalone-tests/lit.cfg

@@ -0,0 +1,4 @@
+import lit.formats
+config.name = 'Standalone tests'
+config.test_format = lit.formats.ShTest()
+config.standalone_tests = True

llvm/utils/lit/tests/Inputs/standalone-tests/true.txt

@@ -0,0 +1 @@
+# RUN: true

llvm/utils/lit/tests/discovery.py

@@ -148,6 +148,36 @@
 # RUN: %{lit} \
 # RUN:     %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check
 
+# Check that a standalone test with no suffixes set is run without any errors.
+#
+# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out
+# RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s
+#
+# CHECK-STANDALONE: PASS: Standalone tests :: true.txt
+
+# Check that an error is produced if suffixes variable is set for a suite with
+# standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err
+# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes
+
+# Check that an error is produced if excludes variable is set for a suite with
+# standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err
+# RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes
+
+# Check that no discovery is done for testsuite with standalone tests.
+#
+# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err
+# RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s
+#
+# CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
+
 # Check that we don't recurse infinitely when loading an site specific test
 # suite located inside the test source root.
 #