[llvm] [lit] Support forcing lexical test order

Add a new --order option to choose between available test orders:
the default "smart" order, predictable "lexical" order or "random"
order.  Default to using lexical order and one job in the lit test
suite.

Differential Revision: https://reviews.llvm.org/D107695

llvm/docs/CommandGuide/lit.rst

@@ -168,15 +168,17 @@ SELECTION OPTIONS
 -----------------
 
 By default, `lit` will run failing tests first, then run tests in descending
-execution time order to optimize concurrency.
+execution time order to optimize concurrency.  The execution order can be
+changed using the :option:`--order` option.
 
 The timing data is stored in the `test_exec_root` in a file named
 `.lit_test_times.txt`. If this file does not exist, then `lit` checks the
 `test_source_root` for the file to optionally accelerate clean builds.
 
 .. option:: --shuffle
 
- Run the tests in a random order, not failing/slowest first.
+ Run the tests in a random order, not failing/slowest first. Deprecated,
+ use :option:`--order` instead.
 
 .. option:: --max-failures N
 
@@ -204,6 +206,19 @@ The timing data is stored in the `test_exec_root` in a file named
  testsuites, for parallel execution on separate machines (say in a large
  testing farm).
 
+.. option:: --order={lexical,random,smart}
+
+ Define the order in which tests are run. The supported values are:
+
+ - lexical - tests will be run in lexical order according to the test file
+   path. This option is useful when predictable test order is desired.
+
+ - random - tests will be run in random order.
+
+ - smart - tests that failed previously will be run first, then the remaining
+   tests, all in descending execution time order. This is the default as it
+   optimizes concurrency.
+
 .. option:: --run-shard=N
 
  Select which shard to run, assuming the ``--num-shards=M`` option was

llvm/utils/lit/lit/cl_arguments.py

@@ -8,9 +8,11 @@
 import lit.util
 
 
+@enum.unique
 class TestOrder(enum.Enum):
-    DEFAULT = enum.auto()
-    RANDOM = enum.auto()
+    LEXICAL = 'lexical'
+    RANDOM = 'random'
+    SMART = 'smart'
 
 
 def parse_args():
@@ -153,11 +155,17 @@ def parse_args():
             metavar="N",
             help="Maximum time to spend testing (in seconds)",
             type=_positive_int)
+    selection_group.add_argument("--order",
+            choices=[x.value for x in TestOrder],
+            default=TestOrder.SMART,
+            help="Test order to use (default: smart)")
     selection_group.add_argument("--shuffle",
-            help="Run tests in random order",
-            action="store_true")
+            dest="order",
+            help="Run tests in random order (DEPRECATED: use --order=random)",
+            action="store_const",
+            const=TestOrder.RANDOM)
     selection_group.add_argument("-i", "--incremental",
-            help="Run failed tests first (DEPRECATED: now always enabled)",
+            help="Run failed tests first (DEPRECATED: use --order=smart)",
             action="store_true")
     selection_group.add_argument("--filter",
             metavar="REGEX",
@@ -218,11 +226,6 @@ def parse_args():
     if opts.incremental:
         print('WARNING: --incremental is deprecated. Failing tests now always run first.')
 
-    if opts.shuffle:
-        opts.order = TestOrder.RANDOM
-    else:
-        opts.order = TestOrder.DEFAULT
-
     if opts.numShards or opts.runShard:
         if not opts.numShards or not opts.runShard:
             parser.error("--num-shards and --run-shard must be used together")

llvm/utils/lit/lit/main.py

@@ -166,11 +166,14 @@ def print_discovered(tests, show_suites, show_tests):
 
 def determine_order(tests, order):
     from lit.cl_arguments import TestOrder
-    if order == TestOrder.RANDOM:
+    enum_order = TestOrder(order)
+    if enum_order == TestOrder.RANDOM:
         import random
         random.shuffle(tests)
+    elif enum_order == TestOrder.LEXICAL:
+        tests.sort(key=lambda t: t.getFullName())
     else:
-        assert order == TestOrder.DEFAULT, 'Unknown TestOrder value'
+        assert enum_order == TestOrder.SMART, 'Unknown TestOrder value'
         tests.sort(key=lambda t: (not t.previous_failure, -t.previous_elapsed, t.getFullName()))
 
 

llvm/utils/lit/tests/allow-retries.py

@@ -4,38 +4,38 @@
 # only succeed the fourth time it is retried.
 #
 # RUN: rm -f %t.counter
-# RUN: %{lit} -j 1 %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
+# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST1 %s
 # CHECK-TEST1: Passed With Retry: 1
 
 # Test that a per-file ALLOW_RETRIES overwrites the config-wide test_retry_attempts property, if any.
 #
 # RUN: rm -f %t.counter
-# RUN: %{lit} -j 1 %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s
+# RUN: %{lit} %{inputs}/allow-retries/succeeds-within-limit.py -Dtest_retry_attempts=2 -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST2 %s
 # CHECK-TEST2: Passed With Retry: 1
 
 # This test does not succeed within the allowed retry limit
 #
-# RUN: not %{lit} -j 1 %{inputs}/allow-retries/does-not-succeed-within-limit.py | FileCheck --check-prefix=CHECK-TEST3 %s
+# RUN: not %{lit} %{inputs}/allow-retries/does-not-succeed-within-limit.py | FileCheck --check-prefix=CHECK-TEST3 %s
 # CHECK-TEST3: Failed Tests (1):
 # CHECK-TEST3: allow-retries :: does-not-succeed-within-limit.py
 
 # This test should be UNRESOLVED since it has more than one ALLOW_RETRIES
 # lines, and that is not allowed.
 #
-# RUN: not %{lit} -j 1 %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s
+# RUN: not %{lit} %{inputs}/allow-retries/more-than-one-allow-retries-lines.py | FileCheck --check-prefix=CHECK-TEST4 %s
 # CHECK-TEST4: Unresolved Tests (1):
 # CHECK-TEST4: allow-retries :: more-than-one-allow-retries-lines.py
 
 # This test does not provide a valid integer to the ALLOW_RETRIES keyword.
 # It should be unresolved.
 #
-# RUN: not %{lit} -j 1 %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s
+# RUN: not %{lit} %{inputs}/allow-retries/not-a-valid-integer.py | FileCheck --check-prefix=CHECK-TEST5 %s
 # CHECK-TEST5: Unresolved Tests (1):
 # CHECK-TEST5: allow-retries :: not-a-valid-integer.py
 
 # This test checks that the config-wide test_retry_attempts property is used
 # when no ALLOW_RETRIES keyword is present.
 #
 # RUN: rm -f %t.counter
-# RUN: %{lit} -j 1 %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s
+# RUN: %{lit} %{inputs}/test_retry_attempts/test.py -Dcounter=%t.counter -Dpython=%{python} | FileCheck --check-prefix=CHECK-TEST6 %s
 # CHECK-TEST6: Passed With Retry: 1

llvm/utils/lit/tests/custom-result-category.py

@@ -1,10 +1,7 @@
 # UNSUPPORTED: system-windows
 # Test lit.main.add_result_category() extension API.
 
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/custom-result-category/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 %{inputs}/custom-result-category | FileCheck %s
+# RUN: not %{lit} %{inputs}/custom-result-category | FileCheck %s
 
 # CHECK: CUSTOM_PASS: custom-result-category :: test1.txt
 # CHECK: CUSTOM_FAILURE: custom-result-category :: test2.txt

llvm/utils/lit/tests/discovery.py

@@ -1,7 +1,7 @@
 # Check the basic discovery process, including a sub-suite.
 #
 # RUN: %{lit} %{inputs}/discovery \
-# RUN:   -j 1 --debug --show-tests --show-suites \
+# RUN:   --debug --show-tests --show-suites \
 # RUN:   -v > %t.out 2> %t.err
 # RUN: FileCheck --check-prefix=CHECK-BASIC-OUT < %t.out %s
 # RUN: FileCheck --check-prefix=CHECK-BASIC-ERR < %t.err %s
@@ -56,7 +56,7 @@
 # RUN: %{lit} \
 # RUN:     %{inputs}/discovery/subdir/test-three.py \
 # RUN:     %{inputs}/discovery/subsuite/test-one.txt \
-# RUN:   -j 1 --show-tests --show-suites -v > %t.out
+# RUN:   --show-tests --show-suites -v > %t.out
 # RUN: FileCheck --check-prefix=CHECK-DIRECT-TEST < %t.out %s
 #
 # CHECK-DIRECT-TEST: -- Available Tests --
@@ -65,7 +65,7 @@
 
 # Check discovery when config files end in .py
 # RUN: %{lit} %{inputs}/py-config-discovery \
-# RUN:   -j 1 --debug --show-tests --show-suites \
+# RUN:   --debug --show-tests --show-suites \
 # RUN:   -v > %t.out 2> %t.err
 # RUN: FileCheck --check-prefix=CHECK-PYCONFIG-OUT < %t.out %s
 # RUN: FileCheck --check-prefix=CHECK-PYCONFIG-ERR < %t.err %s
@@ -95,7 +95,7 @@
 # Check discovery when using an exec path.
 #
 # RUN: %{lit} %{inputs}/exec-discovery \
-# RUN:   -j 1 --debug --show-tests --show-suites \
+# RUN:   --debug --show-tests --show-suites \
 # RUN:   -v > %t.out 2> %t.err
 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-OUT < %t.out %s
 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-ERR < %t.err %s
@@ -128,7 +128,7 @@
 #
 # RUN: %{lit} \
 # RUN:     %{inputs}/exec-discovery/subdir/test-three.py \
-# RUN:   -j 1 --show-tests --show-suites -v > %t.out
+# RUN:   --show-tests --show-suites -v > %t.out
 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-DIRECT-TEST < %t.out %s
 #
 # CHECK-ASEXEC-DIRECT-TEST: -- Available Tests --
@@ -138,42 +138,42 @@
 # indirectly (e.g. when the directory containing the test is specified).
 #
 # RUN: not %{lit} \
-# RUN:     %{inputs}/discovery/test.not-txt -j 1 2>%t.err
+# RUN:     %{inputs}/discovery/test.not-txt 2>%t.err
 # RUN: FileCheck --check-prefix=CHECK-ERROR-INDIRECT-RUN-CHECK < %t.err %s
 #
 # CHECK-ERROR-INDIRECT-RUN-CHECK: error: 'top-level-suite :: test.not-txt' would not be run indirectly
 
 # Check that no error is emitted with --no-indirectly-run-check.
 #
 # RUN: %{lit} \
-# RUN:     %{inputs}/discovery/test.not-txt -j 1 --no-indirectly-run-check
+# RUN:     %{inputs}/discovery/test.not-txt --no-indirectly-run-check
 
 # Check that a standalone test with no suffixes set is run without any errors.
 #
-# RUN: %{lit} %{inputs}/standalone-tests/true.txt -j 1 > %t.out
+# RUN: %{lit} %{inputs}/standalone-tests/true.txt > %t.out
 # RUN: FileCheck --check-prefix=CHECK-STANDALONE < %t.out %s
 #
 # CHECK-STANDALONE: PASS: Standalone tests :: true.txt
 
 # Check that an error is produced if suffixes variable is set for a suite with
 # standalone tests.
 #
-# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes -j 1 2> %t.err
+# RUN: not %{lit} %{inputs}/standalone-tests-with-suffixes 2> %t.err
 # RUN: FileCheck --check-prefixes=CHECK-STANDALONE-SUFFIXES,CHECK-STANDALONE-DISCOVERY < %t.err %s
 #
 # CHECK-STANDALONE-SUFFIXES: standalone_tests set {{.*}} but suffixes
 
 # Check that an error is produced if excludes variable is set for a suite with
 # standalone tests.
 #
-# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes -j 1 2> %t.err
+# RUN: not %{lit} %{inputs}/standalone-tests-with-excludes 2> %t.err
 # RUN: FileCheck --check-prefixes=CHECK-STANDALONE-EXCLUDES,CHECK-STANDALONE-DISCOVERY < %t.err %s
 #
 # CHECK-STANDALONE-EXCLUDES: standalone_tests set {{.*}} but {{.*}} excludes
 
 # Check that no discovery is done for testsuite with standalone tests.
 #
-# RUN: not %{lit} %{inputs}/standalone-tests -j 1 2>%t.err
+# RUN: not %{lit} %{inputs}/standalone-tests 2>%t.err
 # RUN: FileCheck --check-prefix=CHECK-STANDALONE-DISCOVERY < %t.err %s
 #
 # CHECK-STANDALONE-DISCOVERY: error: did not discover any tests for provided path(s)
@@ -183,14 +183,14 @@
 #
 # RUN: %{lit} \
 # RUN:     %{inputs}/exec-discovery-in-tree/obj/ \
-# RUN:   -j 1 --show-tests --show-suites -v > %t.out
+# RUN:   --show-tests --show-suites -v > %t.out
 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s
 #
 # Try it again after cd'ing into the test suite using a short relative path.
 #
 # RUN: cd %{inputs}/exec-discovery-in-tree/obj/
 # RUN: %{lit} . \
-# RUN:   -j 1 --show-tests --show-suites -v > %t.out
+# RUN:   --show-tests --show-suites -v > %t.out
 # RUN: FileCheck --check-prefix=CHECK-ASEXEC-INTREE < %t.out %s
 #
 #      CHECK-ASEXEC-INTREE:   exec-discovery-in-tree-suite - 1 tests

llvm/utils/lit/tests/googletest-discovery-failed.py

@@ -1,6 +1,6 @@
 # Check for correct error message when discovery of tests fails.
 #
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-discovery-failed > %t.cmd.out
+# RUN: not %{lit} -v %{inputs}/googletest-discovery-failed > %t.cmd.out
 # RUN: FileCheck < %t.cmd.out %s
 
 

llvm/utils/lit/tests/googletest-format.py

@@ -1,9 +1,6 @@
 # Check the various features of the GoogleTest format.
 
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/googletest-format/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-format > %t.out
+# RUN: not %{lit} -v %{inputs}/googletest-format > %t.out
 # FIXME: Temporarily dump test output so we can debug failing tests on
 # buildbots.
 # RUN: cat %t.out

llvm/utils/lit/tests/googletest-timeout.py

@@ -6,14 +6,14 @@
 
 # Check that the per test timeout is enforced when running GTest tests.
 #
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
+# RUN: not %{lit} -v %{inputs}/googletest-timeout \
 # RUN:   --filter=InfiniteLoopSubTest --timeout=1 > %t.cmd.out
 # RUN: FileCheck --check-prefix=CHECK-INF < %t.cmd.out %s
 
 # Check that the per test timeout is enforced when running GTest tests via
 # the configuration file
 #
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-timeout \
+# RUN: not %{lit} -v %{inputs}/googletest-timeout \
 # RUN:  --filter=InfiniteLoopSubTest  --param set_timeout=1 \
 # RUN:  > %t.cfgset.out
 # RUN: FileCheck --check-prefix=CHECK-INF < %t.cfgset.out %s
@@ -34,7 +34,7 @@
 # 3600 second timeout.
 ###############################################################################
 
-# RUN: %{lit} -j 1 -v %{inputs}/googletest-timeout \
+# RUN: %{lit} -v %{inputs}/googletest-timeout \
 # RUN:   --filter=QuickSubTest --timeout=3600 > %t.cmd.out
 # RUN: FileCheck --check-prefix=CHECK-QUICK < %t.cmd.out %s
 
@@ -43,7 +43,7 @@
 
 # Test per test timeout via a config file and on the command line.
 # The value set on the command line should override the config file.
-# RUN: %{lit} -j 1 -v %{inputs}/googletest-timeout --filter=QuickSubTest \
+# RUN: %{lit} -v %{inputs}/googletest-timeout --filter=QuickSubTest \
 # RUN:   --param set_timeout=1 --timeout=3600 \
 # RUN:   > %t.cmdover.out 2> %t.cmdover.err
 # RUN: FileCheck --check-prefix=CHECK-QUICK < %t.cmdover.out %s

llvm/utils/lit/tests/googletest-upstream-format.py

@@ -1,9 +1,6 @@
 # Check the various features of the GoogleTest format.
 
-# FIXME: this test depends on order of tests
-# RUN: rm -f %{inputs}/googletest-upstream-format/.lit_test_times.txt
-
-# RUN: not %{lit} -j 1 -v %{inputs}/googletest-upstream-format > %t.out
+# RUN: not %{lit} -v %{inputs}/googletest-upstream-format > %t.out
 # RUN: FileCheck < %t.out %s
 #
 # END.

llvm/utils/lit/tests/ignore-fail.py

@@ -1,8 +1,8 @@
 # Check that --ignore-fail produces exit status 0 despite various kinds of
 # test failures but doesn't otherwise suppress those failures.
 
-# RUN: not %{lit} -j 1 %{inputs}/ignore-fail | FileCheck %s
-# RUN: %{lit} -j 1 --ignore-fail %{inputs}/ignore-fail | FileCheck %s
+# RUN: not %{lit} %{inputs}/ignore-fail | FileCheck %s
+# RUN: %{lit} --ignore-fail %{inputs}/ignore-fail | FileCheck %s
 
 # END.
 

llvm/utils/lit/tests/lit-opts.py

@@ -1,25 +1,25 @@
 # Check cases where LIT_OPTS has no effect.
 #
-# RUN:                 %{lit} -j 1 -s %{inputs}/lit-opts | FileCheck %s
-# RUN: env LIT_OPTS=   %{lit} -j 1 -s %{inputs}/lit-opts | FileCheck %s
-# RUN: env LIT_OPTS=-s %{lit} -j 1 -s %{inputs}/lit-opts | FileCheck %s
+# RUN:                 %{lit} -s %{inputs}/lit-opts | FileCheck %s
+# RUN: env LIT_OPTS=   %{lit} -s %{inputs}/lit-opts | FileCheck %s
+# RUN: env LIT_OPTS=-s %{lit} -s %{inputs}/lit-opts | FileCheck %s
 
 # Check that LIT_OPTS can override command-line options.
 #
 # RUN: env LIT_OPTS=-a \
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts \
+# RUN: %{lit} -s %{inputs}/lit-opts \
 # RUN: | FileCheck -check-prefix=SHOW-ALL -DVAR= %s
 
 # Check that LIT_OPTS understands multiple options with arbitrary spacing.
 #
 # RUN: env LIT_OPTS='-a -v  -Dvar=foobar' \
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts \
+# RUN: %{lit} -s %{inputs}/lit-opts \
 # RUN: | FileCheck -check-prefix=SHOW-ALL -DVAR=foobar %s
 
 # Check that LIT_OPTS parses shell-like quotes and escapes.
 #
 # RUN: env LIT_OPTS='-a   -v -Dvar="foo bar"\ baz' \
-# RUN: %{lit} -j 1 -s %{inputs}/lit-opts \
+# RUN: %{lit} -s %{inputs}/lit-opts \
 # RUN: | FileCheck -check-prefix=SHOW-ALL -DVAR="foo bar baz" %s
 
 # CHECK:      Testing: 1 tests

llvm/utils/lit/tests/lit.cfg

@@ -57,11 +57,14 @@ for attribute in ('llvm_tools_dir', 'lit_tools_dir'):
 # suites in %{inputs}.  This test suite's results are then determined in part
 # by %{lit}'s textual output, which includes the output of FileCheck calls
 # within %{inputs}'s test suites.  Thus, %{lit} clears environment variables
-# that can affect FileCheck's output.
+# that can affect FileCheck's output.  It also includes "--order=lexical -j1"
+# to ensure predictable test order, as it is often required for FileCheck
+# matches.
 config.substitutions.append(('%{inputs}', os.path.join(
     config.test_source_root, 'Inputs')))
-config.substitutions.append(('%{lit}',
-    "{env} %{{python}} {lit}".format(
+config.substitutions.append(('%{lit}', '%{lit-no-order-opt} --order=lexical'))
+config.substitutions.append(('%{lit-no-order-opt}',
+    "{env} %{{python}} {lit} -j1".format(
         env="env -u FILECHECK_OPTS",
         lit=os.path.join(lit_path, 'lit.py'))))
 config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))