diff --git a/llvm/docs/CommandGuide/lit.rst b/llvm/docs/CommandGuide/lit.rst index 15c249d8e6d31..0d1019824c3cb 100644 --- a/llvm/docs/CommandGuide/lit.rst +++ b/llvm/docs/CommandGuide/lit.rst @@ -314,6 +314,11 @@ The timing data is stored in the `test_exec_root` in a file named place of this option, which is especially useful in environments where the call to ``lit`` is issued indirectly. +.. option:: --filter-failed + + Run only those tests that previously failed. Tests that have been newly added + but not yet run are not included. + .. option:: --xfail LIST Treat those tests whose name is in the semicolon separated list ``LIST`` as diff --git a/llvm/utils/lit/lit/cl_arguments.py b/llvm/utils/lit/lit/cl_arguments.py index 8238bc42395af..231d180b63cbd 100644 --- a/llvm/utils/lit/lit/cl_arguments.py +++ b/llvm/utils/lit/lit/cl_arguments.py @@ -302,6 +302,12 @@ def parse_args(): help="Filter out tests with paths matching the given regular expression", default=os.environ.get("LIT_FILTER_OUT", "^$"), ) + selection_group.add_argument( + "--filter-failed", + dest="filterFailed", + help="Only run tests which failed in the previous run", + action="store_true", + ) selection_group.add_argument( "--xfail", metavar="LIST", diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py index a585cc0abdd48..6c650724bb33d 100755 --- a/llvm/utils/lit/lit/main.py +++ b/llvm/utils/lit/lit/main.py @@ -90,6 +90,9 @@ def main(builtin_params={}): and not opts.filter_out.search(t.getFullName()) ] + if opts.filterFailed: + selected_tests = [t for t in selected_tests if t.previous_failure] + if not selected_tests: sys.stderr.write( "error: filter did not match any tests " diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt new file mode 100644 index 0000000000000..15eb81a5f5e95 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/filter-failed/fail.txt @@ -0,0 +1 @@ +RUN: false diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg new file mode 100644 index 0000000000000..5aee4eb132d6e --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/filter-failed/lit.cfg @@ -0,0 +1,7 @@ +import lit.formats + +config.name = "filter-failed" +config.suffixes = [".txt"] +config.test_format = lit.formats.ShTest() +config.test_source_root = None +config.test_exec_root = None diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt new file mode 100644 index 0000000000000..18efe9e49e95b --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/filter-failed/pass.txt @@ -0,0 +1 @@ +RUN: true diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt b/llvm/utils/lit/tests/Inputs/filter-failed/unresolved.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt new file mode 100644 index 0000000000000..6814cda401483 --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/filter-failed/xfail.txt @@ -0,0 +1,2 @@ +RUN: false +XFAIL: * diff --git a/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt new file mode 100644 index 0000000000000..66b8a6a5a187c --- /dev/null +++ b/llvm/utils/lit/tests/Inputs/filter-failed/xpass.txt @@ -0,0 +1,2 @@ +RUN: true +XFAIL: * diff --git a/llvm/utils/lit/tests/filter-failed-delete.py b/llvm/utils/lit/tests/filter-failed-delete.py new file mode 100644 index 0000000000000..68bb840425ea3 --- /dev/null +++ b/llvm/utils/lit/tests/filter-failed-delete.py @@ -0,0 +1,16 @@ +# Shows behaviour when a previously failed test was deleted +# before running with --filter-failed. + +# RUN: rm -rf %t +# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t +# +# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST +# +# RUN: rm %t%{fs-sep}fail.txt +# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-SECOND + +# CHECK-FIRST: Testing: 5 tests +# CHECK-FIRST: FAIL: filter-failed :: fail.txt + +# CHECK-SECOND: Testing: 2 of 4 tests +# CHECK-SECOND-NOT: filter-failed :: fail.txt diff --git a/llvm/utils/lit/tests/filter-failed-rerun.py b/llvm/utils/lit/tests/filter-failed-rerun.py new file mode 100644 index 0000000000000..7db12c26a02ff --- /dev/null +++ b/llvm/utils/lit/tests/filter-failed-rerun.py @@ -0,0 +1,18 @@ +# Checks that --filter-failed won't re-run tests that have passed +# since the last time --filter-failed was run. + +# RUN: rm -rf %t +# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t +# +# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-FIRST +# +# RUN: cp %t%{fs-sep}pass.txt %t%{fs-sep}fail.txt +# RUN: not %{lit} %t | FileCheck %s --check-prefix=CHECK-SECOND +# RUN: not %{lit} --filter-failed %t | FileCheck %s --check-prefix=CHECK-THIRD + +# CHECK-FIRST: FAIL: filter-failed :: fail.txt + +# CHECK-SECOND: PASS: filter-failed :: fail.txt + +# CHECK-THIRD: Testing: 2 of 5 tests +# CHECK-THIRD-NOT: filter-failed :: fail.txt diff --git a/llvm/utils/lit/tests/filter-failed.py b/llvm/utils/lit/tests/filter-failed.py new file mode 100644 index 0000000000000..3c1c6fe010077 --- /dev/null +++ b/llvm/utils/lit/tests/filter-failed.py @@ -0,0 +1,23 @@ +# Checks that --filter-failed only runs tests that previously failed. + +# RUN: rm -rf %t +# RUN: cp -r %{inputs}%{fs-sep}filter-failed %t +# +# RUN: not %{lit} %t +# +# RUN: echo "RUN: false" > %t%{fs-sep}new-fail.txt +# RUN: echo "RUN: true" > %t%{fs-sep}new-pass.txt +# +# RUN: not %{lit} --filter-failed %t | FileCheck %s + +# CHECK: Testing: 3 of 7 tests +# CHECK-DAG: FAIL: filter-failed :: fail.txt +# CHECK-DAG: UNRESOLVED: filter-failed :: unresolved.txt +# CHECK-DAG: XPASS: filter-failed :: xpass.txt + +# CHECK: Testing Time: +# CHECK: Total Discovered Tests: +# CHECK-NEXT: Excluded : 4 {{\([0-9]*\.[0-9]*%\)}} +# CHECK-NEXT: Unresolved : 1 {{\([0-9]*\.[0-9]*%\)}} +# CHECK-NEXT: Failed : 1 {{\([0-9]*\.[0-9]*%\)}} +# CHECK-NEXT: Unexpectedly Passed: 1 {{\([0-9]*\.[0-9]*%\)}}