Reapply "[llvm][lit] Add option to run only the failed tests" (#171588)

This reverts commit 3847648.

Relands #158043 which got
auto-merged on a revision which wasn't approved.

The only addition to the approved version was that we adjust how we set
the time for failed tests. We used to just assign it the negative value
of the elapsed time. But if the test failed with `0` seconds (which some
of the new tests do), we would mark it `-0`. But the check for whether
something failed checks for `time < 0`. That messed with the new
`--filter-failed` option of this PR. This was only an issue on Windows
CI, but presumably can happen on any platform. Happy to do this in a
separate PR.

---- Original PR

This patch adds a new --filter-failed option to llvm-lit, which when
set, will only run the tests that have previously failed.

Author: Michael137

llvm/docs/CommandGuide/lit.rst

@@ -314,6 +314,11 @@ The timing data is stored in the `test_exec_root` in a file named
   place of this option, which is especially useful in environments where the
   call to ``lit`` is issued indirectly.
 
+.. option:: --filter-failed
+
+  Run only those tests that previously failed. Tests that have been newly added
+  but not yet run are not included.
+
 .. option:: --xfail LIST
 
   Treat those tests whose name is in the semicolon separated list ``LIST`` as

llvm/utils/lit/lit/TestTimes.py

@@ -22,7 +22,12 @@ def record_test_times(tests, lit_config):
             continue
         if not t.suite.exec_root in times_by_suite:
             times_by_suite[t.suite.exec_root] = read_test_times(t.suite)
-        time = -t.result.elapsed if t.isFailure() else t.result.elapsed
+
+        # Mark the elapsed time for failed tests as negative so LIT can distingiush failed from
+        # successful test runs just based on the time value. For this heuristic to work for tests
+        # whose elapsed time is '0', we set it to a small negative constant.
+        time = min(-t.result.elapsed, -1.0e-6) if t.isFailure() else t.result.elapsed
+
         # The "path" here is only used as a key into a dictionary. It is never
         # used as an actual path to a filesystem API, therefore we use '/' as
         # the canonical separator so that Unix and Windows machines can share

llvm/utils/lit/lit/cl_arguments.py

@@ -423,6 +423,12 @@ def parse_args():
         help="Filter out tests with paths matching the given regular expression",
         default=os.environ.get("LIT_FILTER_OUT", "^$"),
     )
+    selection_group.add_argument(
+        "--filter-failed",
+        dest="filterFailed",
+        help="Only run tests which failed in the previous run",
+        action="store_true",
+    )
     selection_group.add_argument(
         "--xfail",
         metavar="LIST",

llvm/utils/lit/lit/main.py

@@ -90,6 +90,9 @@ def main(builtin_params={}):
         and not opts.filter_out.search(t.getFullName())
     ]
 
+    if opts.filterFailed:
+        selected_tests = [t for t in selected_tests if t.previous_failure]
+
     if not selected_tests:
         sys.stderr.write(
             "error: filter did not match any tests "

llvm/utils/lit/tests/Inputs/filter-failed/fail.txt

@@ -0,0 +1 @@
+RUN: false

llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg

@@ -0,0 +1,7 @@
+import lit.formats
+
+config.name = "filter-failed"
+config.suffixes = [".txt"]
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None

llvm/utils/lit/tests/Inputs/filter-failed/pass.txt

@@ -0,0 +1 @@
+RUN: true

llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt

@@ -0,0 +1,2 @@
+RUN: false
+XFAIL: *

llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt

@@ -0,0 +1,2 @@
+RUN: true
+XFAIL: *