Merge bitcoin#10374: qa: Warn when specified test is not found

fac79e4 qa: Warn when specified test is not found (MarcoFalke)

Tree-SHA512: d11ecdde275309b12e23155f6cd8e26c99217436b5094a70dd51b95ae7688754227628dd9a801eb6a52ff3ebea4420938e2fc8e9dc9cd77a4dd5c28d2b822354

test/README.md

@@ -34,7 +34,7 @@ You can run any single test by calling
 
     test/functional/test_runner.py <testname>
 
-Or you can run any combination of tests by calling
+Or you can run any combination (incl. duplicates) of tests by calling
 
     test/functional/test_runner.py <testname1> <testname2> <testname3> ...
 

test/functional/test_runner.py

@@ -176,16 +176,16 @@ def main():
     Help text and arguments for individual test script:''',
                                      formatter_class=argparse.RawTextHelpFormatter)
     parser.add_argument('--coverage', action='store_true', help='generate a basic coverage report for the RPC interface')
-    parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude. Do not include the .py extension in the name.')
+    parser.add_argument('--exclude', '-x', help='specify a comma-seperated-list of scripts to exclude.')
     parser.add_argument('--extended', action='store_true', help='run the extended test suite in addition to the basic tests')
     parser.add_argument('--force', '-f', action='store_true', help='run tests even on platforms where they are disabled by default (e.g. windows).')
     parser.add_argument('--help', '-h', '-?', action='store_true', help='print help text and exit')
     parser.add_argument('--jobs', '-j', type=int, default=4, help='how many test scripts to run in parallel. Default=4.')
     parser.add_argument('--keepcache', '-k', action='store_true', help='the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun.')
     args, unknown_args = parser.parse_known_args()
 
-    # Create a set to store arguments and create the passon string
-    tests = set(arg for arg in unknown_args if arg[:2] != "--")
+    # args to be passed on always start with two dashes; tests are the remaining unknown args
+    tests = [arg for arg in unknown_args if arg[:2] != "--"]
     passon_args = [arg for arg in unknown_args if arg[:2] == "--"]
 
     # Read config generated by configure.
@@ -214,8 +214,13 @@ def main():
     if tests:
         # Individual tests have been specified. Run specified tests that exist
         # in the ALL_SCRIPTS list. Accept the name with or without .py extension.
-        test_list = [t for t in ALL_SCRIPTS if
-                (t in tests or re.sub(".py$", "", t) in tests)]
+        tests = [re.sub("\.py$", "", t) + ".py" for t in tests]
+        test_list = []
+        for t in tests:
+            if t in ALL_SCRIPTS:
+                test_list.append(t)
+            else:
+                print("{}WARNING!{} Test '{}' not found in full test list.".format(BOLD[1], BOLD[0], t))
     else:
         # No individual tests have been specified.
         # Run all base tests, and optionally run extended tests.
@@ -227,9 +232,12 @@ def main():
 
     # Remove the test cases that the user has explicitly asked to exclude.
     if args.exclude:
-        for exclude_test in args.exclude.split(','):
-            if exclude_test + ".py" in test_list:
-                test_list.remove(exclude_test + ".py")
+        tests_excl = [re.sub("\.py$", "", t) + ".py" for t in args.exclude.split(',')]
+        for exclude_test in tests_excl:
+            if exclude_test in test_list:
+                test_list.remove(exclude_test)
+            else:
+                print("{}WARNING!{} Test '{}' not found in current test list.".format(BOLD[1], BOLD[0], exclude_test))
 
     if not test_list:
         print("No valid test scripts specified. Check that your test is in one "