From 160a1cdf3859ce8254f9f5c05e014dc5aa0deea2 Mon Sep 17 00:00:00 2001 From: Barry Deeney Date: Wed, 5 Jun 2019 21:59:02 +0800 Subject: [PATCH 1/2] Added FAILFAST for functional test suite --- .travis/test_06_script_b.sh | 2 +- qa/pull-tester/rpc-tests.py | 16 +++++++++++++--- qa/rpc-tests/test_framework/test_framework.py | 1 + qa/rpc-tests/test_framework/util.py | 3 +-- qa/rpc-tests/wallet-hd.py | 2 +- 5 files changed, 17 insertions(+), 7 deletions(-) diff --git a/.travis/test_06_script_b.sh b/.travis/test_06_script_b.sh index 79bf261d9..939ac2032 100755 --- a/.travis/test_06_script_b.sh +++ b/.travis/test_06_script_b.sh @@ -16,6 +16,6 @@ fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC LOCAL_NTP=1 ./qa/pull-tester/rpc-tests.py -parallel=1 --coverage + DOCKER_EXEC LOCAL_NTP=1 ./qa/pull-tester/rpc-tests.py -parallel=1 --coverage --failfast END_FOLD fi diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 0ddc44e58..adcc964be 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -28,6 +28,7 @@ import subprocess import tempfile import re +import logging sys.path.append("qa/pull-tester/") from tests_config import * @@ -50,7 +51,8 @@ if 'ENABLE_ZMQ' not in vars(): ENABLE_ZMQ=0 -ENABLE_COVERAGE=0 +ENABLE_COVERAGE = False +FAILFAST = False #Create a set to store arguments and create the passon string opts = set() @@ -66,7 +68,9 @@ print_help = True break if arg == '--coverage': - ENABLE_COVERAGE = 1 + ENABLE_COVERAGE = True + elif arg == '--failfast': + FAILFAST = True elif PASSON_REGEX.match(arg): passon_args.append(arg) elif PARALLEL_REGEX.match(arg): @@ -181,7 +185,7 @@ 'mnemonic.py', 'sendtoaddress.py', 'stakeimmaturebalance.py', - 'rpc-help.py', + 'rpc-help.py', ] #if ENABLE_ZMQ: # testScripts.append('zmq_test.py') @@ -257,6 +261,12 @@ def runtests(): print('stderr:\n' if not stderr == '' else '', stderr) results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + + # Check if we need to quit + if FAILFAST and not passed: + logging.debug("Early exiting after test failure") + break + results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] print(results) print("\nRuntime: %s s" % (int(time.time() - time0))) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index ce6aa3b0c..53ea3046f 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -120,6 +120,7 @@ def main(self): help="The seed to use for assigning port numbers (default: current process id)") parser.add_option("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") + self.add_options(parser) (self.options, self.args) = parser.parse_args() diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 505bc7b8b..af2dd9b63 100755 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -51,7 +51,7 @@ class PortSeed: def enable_mocktime(): #For backwared compatibility of the python scripts - #with previous versions of the cache, set MOCKTIME + #with previous versions of the cache, set MOCKTIME #to Jan 1, 2014 + (201 * 10 * 60) global MOCKTIME MOCKTIME = 1388534400 + (201 * 10 * 60) @@ -68,7 +68,6 @@ def enable_coverage(dirname): global COVERAGE_DIR COVERAGE_DIR = dirname - def get_rpc_proxy(url, node_number, timeout=None): """ Args: diff --git a/qa/rpc-tests/wallet-hd.py b/qa/rpc-tests/wallet-hd.py index d32e5043e..47275a292 100755 --- a/qa/rpc-tests/wallet-hd.py +++ b/qa/rpc-tests/wallet-hd.py @@ -38,7 +38,7 @@ def run_test (self): non_hd_add = self.nodes[0].getnewaddress() self.nodes[1].importprivkey(self.nodes[0].dumpprivkey(non_hd_add)) - # This should be enough to keep the master key and the non-HD key + # This should be enough to keep the master key and the non-HD key self.nodes[1].backupwallet(tmpdir + "hd.bak") #self.nodes[1].dumpwallet(tmpdir + "hd.dump") From 206c305828218403c5c2c6a4d72f3f92927fe4e7 Mon Sep 17 00:00:00 2001 From: Barry Deeney Date: Wed, 5 Jun 2019 22:58:37 +0800 Subject: [PATCH 2/2] Updated to use 3 threads for test suite --- .travis/test_06_script_b.sh | 2 +- qa/pull-tester/rpc-tests.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.travis/test_06_script_b.sh b/.travis/test_06_script_b.sh index 939ac2032..5f2b855b5 100755 --- a/.travis/test_06_script_b.sh +++ b/.travis/test_06_script_b.sh @@ -16,6 +16,6 @@ fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC LOCAL_NTP=1 ./qa/pull-tester/rpc-tests.py -parallel=1 --coverage --failfast + DOCKER_EXEC LOCAL_NTP=1 ./qa/pull-tester/rpc-tests.py -parallel=3 --coverage --failfast END_FOLD fi diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index adcc964be..988fa5793 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -28,7 +28,6 @@ import subprocess import tempfile import re -import logging sys.path.append("qa/pull-tester/") from tests_config import * @@ -264,7 +263,7 @@ def runtests(): # Check if we need to quit if FAILFAST and not passed: - logging.debug("Early exiting after test failure") + print("Early exiting after test failure") break results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0]