From 2f41326102157dd5e935029e2bc5d3a996f7a895 Mon Sep 17 00:00:00 2001 From: Jonas Schnelli Date: Fri, 6 May 2016 12:40:12 +0200 Subject: [PATCH 01/13] Autofind rpc tests --srcdir --- qa/rpc-tests/test_framework/test_framework.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index da62e4f7bec..29f4b0f093e 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -135,7 +135,7 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): help="Leave bitcoinds and test.* datadir on exit or error") parser.add_option("--noshutdown", dest="noshutdown", default=False, action="store_true", help="Don't stop bitcoinds after the test execution") - parser.add_option("--srcdir", dest="srcdir", default="../../src", + parser.add_option("--srcdir", dest="srcdir", default=os.path.normpath(os.path.dirname(os.path.realpath(__file__))+"/../../../src"), help="Source directory containing bitcoind/bitcoin-cli (default: %default)") parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"), help="Root directory for datadirs") From 1c602d221d9ef0cd1c335e4ea937f4034e4a7471 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sat, 30 Apr 2016 14:55:31 +0200 Subject: [PATCH 02/13] [qa] Stop other nodes, even when one fails to stop --- qa/rpc-tests/test_framework/util.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 60eb1de4904..48b2ad3a0af 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -19,6 +19,7 @@ from decimal import Decimal, ROUND_DOWN import decimal import json +import http.client import random import shutil import subprocess @@ -375,13 +376,19 @@ def log_filename(dirname, n_node, logname): return os.path.join(dirname, "node"+str(n_node), "regtest", logname) def stop_node(node, i): - node.stop() + try: + node.stop() + except http.client.CannotSendRequest as e: + print("WARN: Unable to stop node: " + repr(e)) bitcoind_processes[i].wait() del bitcoind_processes[i] def stop_nodes(nodes): for node in nodes: - node.stop() + try: + node.stop() + except http.client.CannotSendRequest as e: + print("WARN: Unable to stop node: " + repr(e)) del nodes[:] # Emptying array closes connections as a side effect def set_node_times(nodes, t): From 9ace0e95091d09dcd1f6b88725435b957c1b4017 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 17 May 2017 18:58:09 +0200 Subject: [PATCH 03/13] [qa] Refactor test_framework and pull tester * log to stdout * increase range for p2p and rpc ports * Stop nodes on CTRL+C (partial cherry pick, avoid to apply on rpc-tests.py) --- qa/rpc-tests/test_framework/test_framework.py | 4 +- qa/rpc-tests/test_framework/util.py | 51 +++++-------------- qa/rpc-tests/walletbackup.py | 2 +- 3 files changed, 16 insertions(+), 41 deletions(-) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 29f4b0f093e..49a4dcc88dd 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -159,7 +159,7 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): if self.options.trace_rpc: import logging - logging.basicConfig(level=logging.DEBUG) + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) if self.options.coveragedir: enable_coverage(self.options.coveragedir) @@ -207,6 +207,8 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): typ, value, tb = sys.exc_info() traceback.print_tb(tb) if self.drop_to_pdb: pdb.post_mortem(tb) + except KeyboardInterrupt as e: + print("Exiting after " + repr(e)) if not self.options.noshutdown: print("Stopping nodes") diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 48b2ad3a0af..206d151df9b 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -39,6 +39,12 @@ BTC = 100000000 mBTC = 100000 uBTC = 100 +# The maximum number of nodes a single test can spawn +MAX_NODES = 8 +# Don't assign rpc or p2p ports lower than this +PORT_MIN = 11000 +# The number of ports to "reserve" for p2p and rpc, each +PORT_RANGE = 5000 #Set Mocktime default to OFF. #MOCKTIME is only needed for scripts that use the @@ -93,44 +99,11 @@ def get_rpc_proxy(url, node_number, timeout=None): def p2p_port(n): - #If port is already defined then return port - if os.getenv("node" + str(n)): - return int(os.getenv("node" + str(n))) - #If no port defined then find an available port - if n == 0: - port = 11000 + n + os.getpid()%990 - else: - port = int(os.getenv("node" + str(n-1))) + 1 - from subprocess import check_output - netStatOut = check_output(["netstat", "-n"]) - for portInUse in re.findall(b"tcp.*?:(11\d\d\d)",netStatOut.lower()): - #print portInUse - if port == int(portInUse): - port += 1 - os.environ["node" + str(n)] = str(port) - - #print "port node " + str(n) + " is " + str(port) - return int(port) + assert(n <= MAX_NODES) + return PORT_MIN + n + (MAX_NODES * os.getpid()) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): - #If port is already defined then return port - if os.getenv("rpcnode" + str(n)): - return int(os.getenv("rpcnode" + str(n))) - #If no port defined then find an available port - if n == 0: - port = 12000 + n + os.getpid()%990 - else: - port = int(os.getenv("rpcnode" + str(n-1))) + 1 - from subprocess import check_output - netStatOut = check_output(["netstat", "-n"]) - for portInUse in re.findall(b"tcp.*?:(12\d\d\d)",netStatOut.lower()): - #print portInUse - if port == int(portInUse): - port += 1 - os.environ["rpcnode" + str(n)] = str(port) - - #print "port rpcnode " + str(n) + " is " + str(port) - return int(port) + return PORT_MIN + PORT_RANGE + n + (MAX_NODES * os.getpid()) % (PORT_RANGE -1 - MAX_NODES) def check_json_precision(): """Make sure json library being used does not lose precision converting BTC values""" @@ -361,8 +334,8 @@ def start_nodes(num_nodes, dirname, extra_args=None, rpchost=None, binary=None,t """ Start multiple bitcoinds, return RPC connections to them """ - if extra_args is None: extra_args = [ None for i in range(num_nodes) ] - if binary is None: binary = [ None for i in range(num_nodes) ] + if extra_args is None: extra_args = [ None for _ in range(num_nodes) ] + if binary is None: binary = [ None for _ in range(num_nodes) ] rpcs = [] try: for i in range(num_nodes): @@ -419,7 +392,7 @@ def interconnect_nodes(nodes): for to in nodes: if frm == to: continue up = urlparse.urlparse(to.url) - ip_port = up.hostname + ":" + str(up.port-1000) # this is the RPC port but we want the p2p port so -1000 + ip_port = up.hostname + ":" + str(up.port - PORT_RANGE) # this is the RPC port but we want the p2p port so -1000 frm.addnode(ip_port, "onetry") def find_output(node, txid, amount): diff --git a/qa/rpc-tests/walletbackup.py b/qa/rpc-tests/walletbackup.py index 1978568d594..c62d3db192c 100755 --- a/qa/rpc-tests/walletbackup.py +++ b/qa/rpc-tests/walletbackup.py @@ -39,7 +39,7 @@ from test_framework.util import * from random import randint import logging -logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO) +logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO, stream=sys.stdout) class WalletBackupTest(BitcoinTestFramework): From c800f7ab75e2ff032d6c665a58acf758a5c843ba Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 7 Nov 2016 23:00:54 +0100 Subject: [PATCH 04/13] [qa] test_framework: Exit when tmpdir exists --- qa/rpc-tests/test_framework/test_framework.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 49a4dcc88dd..915de041602 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -170,8 +170,7 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): success = False try: - if not os.path.isdir(self.options.tmpdir): - os.makedirs(self.options.tmpdir) + os.makedirs(self.options.tmpdir, exist_ok=False) # Not pretty but, I changed the function signature # of setup_chain to allow customization of the setup. @@ -182,11 +181,8 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): self.setup_chain(bitcoinConfDict, wallets) self.setup_network() - self.run_test() - success = True - except JSONRPCException as e: print("JSONRPC error: "+e.error['message']) typ, value, tb = sys.exc_info() From 39cf113d1b4808a505cf70f90d4ee142a2785996 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Thu, 18 May 2017 14:36:16 +0200 Subject: [PATCH 05/13] [qa] pull-tester: Run rpc test in parallel manual cherry pick for Core fa494dec --- qa/pull-tester/rpc-tests.py | 119 +++++++++++++++++++++-------------- qa/rpc-tests/create_cache.py | 29 +++++++++ 2 files changed, 100 insertions(+), 48 deletions(-) create mode 100755 qa/rpc-tests/create_cache.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index fd5ae5742de..91bfe2fdca3 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -66,6 +66,9 @@ passOn = "" showHelp = False # if we need to print help p = re.compile("^--") +p_parallel = re.compile('^-parallel=') +run_parallel = 4 + # some of the single-dash options applicable only to this runner script # are also allowed in double-dash format (but are not passed on to the # test scripts themselves) @@ -119,6 +122,9 @@ def option_passed(option_without_dashes): passOn += " " + arg # add it to double_opts only for validation double_opts.add(arg) + elif p_parallel.match(arg): + run_parallel = int(arg.split(sep='=', maxsplit=1)[1]) + else: # this is for single-dash options only # they are interpreted only by this script @@ -347,52 +353,32 @@ def runtests(): trimmed_tests_to_run.append(t) tests_to_run = trimmed_tests_to_run - # now run the tests - p = re.compile(" -h| --help| -help") - for t in tests_to_run: - scriptname=re.sub(".py$", "", str(t).split(' ')[0]) - fullscriptcmd=str(t) - - # print the wrapper-specific help options - if showHelp: - show_wrapper_options() - - if bad_opts_found: - if not ' --help' in passOn: - passOn += ' --help' - - if len(double_opts): - for additional_opt in fullscriptcmd.split(' ')[1:]: - if additional_opt not in double_opts: - continue - - #if fullscriptcmd not in execution_time.keys(): - if 1: - if t in testScripts: - print("Running testscript %s%s%s ..." % (bold[1], t, bold[0])) - else: - print("Running 2nd level testscript " - + "%s%s%s ..." % (bold[1], t, bold[0])) - - time0 = time.time() - test_passed[fullscriptcmd] = False - try: - subprocess.check_call( - rpcTestDir + repr(t) + flags, shell=True) - test_passed[fullscriptcmd] = True - except subprocess.CalledProcessError as e: - print( e ) - test_failure_info[fullscriptcmd] = e - - # exit if help was called - if showHelp: - sys.exit(0) - else: - execution_time[fullscriptcmd] = int(time.time() - time0) - print("Duration: %s s\n" % execution_time[fullscriptcmd]) - - else: - print("Skipping extended test name %s - already executed in regular\n" % scriptname) + if len(tests_to_run) > 1: + # Populate cache + subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + [flags]) + + tests_to_run = list(map(str,tests_to_run)) + max_len_name = len(max(tests_to_run, key=len)) + time_sum = 0 + time0 = time.time() + job_queue = RPCTestHandler(run_parallel, tests_to_run, flags) + results = BOLD[1] + "%s | %s | %s\n\n" % ("TEST".ljust(max_len_name), "PASSED", "DURATION") + BOLD[0] + all_passed = True + + for _ in range(len(tests_to_run)): + (name, stdout, stderr, passed, duration) = job_queue.get_next() + all_passed = all_passed and passed + time_sum += duration + + print('\n' + BOLD[1] + name + BOLD[0] + ":") + print(stdout) + print('stderr:\n' if not stderr == '' else '', stderr) + results += "%s | %s | %s s\n" % (name.ljust(max_len_name), str(passed).ljust(6), duration) + print("Pass: %s%s%s, Duration: %s s\n" % (BOLD[1], passed, BOLD[0], duration)) + + results += BOLD[1] + "\n%s | %s | %s s (accumulated)" % ("ALL".ljust(max_len_name), str(all_passed).ljust(6), time_sum) + BOLD[0] + print(results) + print("\nRuntime: %s s" % (int(time.time() - time0))) if coverage: coverage.report_rpc_coverage() @@ -421,12 +407,49 @@ def runtests(): print("%d test(s) disabled / %d test(s) skipped due to platform" % (len(disabled), len(skipped))) # signal that tests have failed using exit code - if list(test_passed.values()).count(False): - sys.exit(1) + sys.exit(not all_passed) else: print("No rpc tests to run. Wallet, utils, and bitcoind must all be enabled") +class RPCTestHandler: + """ + Trigger the testscrips passed in via the list. + """ + + def __init__(self, num_tests_parallel, test_list=None, flags=None): + assert(num_tests_parallel >= 1) + self.num_jobs = num_tests_parallel + self.test_list = test_list + self.flags = flags + self.num_running = 0 + self.jobs = [] + + def get_next(self): + while self.num_running < self.num_jobs and self.test_list: + # Add tests + self.num_running += 1 + t = self.test_list.pop(0) + self.jobs.append((t, + time.time(), + subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags.split(), + universal_newlines=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE))) + if not self.jobs: + raise IndexError('%s from empty list' % __name__) + while True: + # Return first proc that finishes + time.sleep(.5) + for j in self.jobs: + (name, time0, proc) = j + if proc.poll() is not None: + (stdout, stderr) = proc.communicate(timeout=3) + passed = stderr == "" and proc.returncode == 0 + self.num_running -= 1 + self.jobs.remove(j) + return name, stdout, stderr, passed, int(time.time() - time0) + print('.', end='', flush=True) class RPCCoverage(object): """ diff --git a/qa/rpc-tests/create_cache.py b/qa/rpc-tests/create_cache.py new file mode 100755 index 00000000000..1ace6310d0d --- /dev/null +++ b/qa/rpc-tests/create_cache.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# Copyright (c) 2016 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Helper script to create the cache +# (see BitcoinTestFramework.setup_chain) +# + +from test_framework.test_framework import BitcoinTestFramework + +class CreateCache(BitcoinTestFramework): + + def __init__(self): + super().__init__() + + # Test network and test nodes are not required: + self.num_nodes = 0 + self.nodes = [] + + def setup_network(self): + pass + + def run_test(self): + pass + +if __name__ == '__main__': + CreateCache().main() From 03a77fe8f622016865a759e8b021fedb6de6a9e4 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 9 May 2016 19:55:49 +0200 Subject: [PATCH 06/13] [qa] Add option --portseed to test_framework --- qa/pull-tester/rpc-tests.py | 3 ++- qa/rpc-tests/test_framework/test_framework.py | 13 +++++++++---- qa/rpc-tests/test_framework/util.py | 9 +++++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 91bfe2fdca3..13306efc172 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -430,9 +430,10 @@ def get_next(self): # Add tests self.num_running += 1 t = self.test_list.pop(0) + port_seed = ["--portseed=%s" % len(self.test_list)] self.jobs.append((t, time.time(), - subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags.split(), + subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags.split() + port_seed, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE))) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 915de041602..a3edab248cf 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -6,7 +6,8 @@ # Base class for RPC testing -# Add python-bitcoinrpc to module search path: +import logging +import optparse import os import sys import time # BU added @@ -28,8 +29,9 @@ enable_coverage, check_json_precision, initialize_chain_clean, + PortSeed, ) -from .authproxy import AuthServiceProxy, JSONRPCException +from .authproxy import JSONRPCException class BitcoinTestFramework(object): @@ -121,7 +123,7 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): """ argsOverride: pass your own values for sys.argv in this field (or pass None) to use sys.argv bitcoinConfDict: Pass a dictionary of values you want written to bitcoin.conf. If you have a key with multiple values, pass a list of the values as the value, for example: - { "debug":["net","blk","thin","lck","mempool","req","bench","evict"] } + { "debug":["net","blk","thin","lck","mempool","req","bench","evict"] } This framework provides values for the necessary fields (like regtest=1). But you can override these defaults by setting them in this dictionary. @@ -141,6 +143,8 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): help="Root directory for datadirs") parser.add_option("--tracerpc", dest="trace_rpc", default=False, action="store_true", help="Print out all RPC calls as they are made") + parser.add_option("--portseed", dest="port_seed", default=os.getpid(), type='int', + help="The seed to use for assigning port numbers (default: current process id)") parser.add_option("--coveragedir", dest="coveragedir", help="Write tested RPC commands into this directory") # BU: added for tests using randomness (e.g. excessive.py) @@ -158,12 +162,13 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): print("Random seed: %s" % self.randomseed) if self.options.trace_rpc: - import logging logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) if self.options.coveragedir: enable_coverage(self.options.coveragedir) + PortSeed.n = self.options.port_seed + os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH'] check_json_precision() diff --git a/qa/rpc-tests/test_framework/util.py b/qa/rpc-tests/test_framework/util.py index 206d151df9b..07fad09afd6 100644 --- a/qa/rpc-tests/test_framework/util.py +++ b/qa/rpc-tests/test_framework/util.py @@ -46,6 +46,11 @@ # The number of ports to "reserve" for p2p and rpc, each PORT_RANGE = 5000 + +class PortSeed: + # Must be initialized with a unique integer for each process + n = None + #Set Mocktime default to OFF. #MOCKTIME is only needed for scripts that use the #cached version of the blockchain. If the cached @@ -100,10 +105,10 @@ def get_rpc_proxy(url, node_number, timeout=None): def p2p_port(n): assert(n <= MAX_NODES) - return PORT_MIN + n + (MAX_NODES * os.getpid()) % (PORT_RANGE - 1 - MAX_NODES) + return PORT_MIN + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def rpc_port(n): - return PORT_MIN + PORT_RANGE + n + (MAX_NODES * os.getpid()) % (PORT_RANGE -1 - MAX_NODES) + return PORT_MIN + PORT_RANGE + n + (MAX_NODES * PortSeed.n) % (PORT_RANGE - 1 - MAX_NODES) def check_json_precision(): """Make sure json library being used does not lose precision converting BTC values""" From 37eba151a51fc4804bcfbc5ff30f4d62f706dc85 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Sun, 15 May 2016 11:06:23 +0200 Subject: [PATCH 07/13] [qa] test_framework: Append portseed to tmpdir This makes it possible to specify a tmpdir while running tests in parallel --- qa/rpc-tests/test_framework/test_framework.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index a3edab248cf..594d4571bce 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -161,6 +161,8 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): random.seed(self.randomseed) print("Random seed: %s" % self.randomseed) + self.options.tmpdir += '/' + str(self.options.port_seed) + if self.options.trace_rpc: logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) From 32ed8fe4309e3485a6059db8d61b4e2ada8a346b Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Mon, 7 Nov 2016 22:33:22 +0100 Subject: [PATCH 08/13] [qa] rpc-tests: Apply random offset to portseed This helps to skip over resources, which are blocked by regtest bitcoind zombie nodes Github-Pull: #9098 Rebased-From: fab0f07dec8d6e21ab70843fdce101f1703588fd --- qa/pull-tester/rpc-tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 13306efc172..4a1b7debb61 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -423,6 +423,10 @@ def __init__(self, num_tests_parallel, test_list=None, flags=None): self.test_list = test_list self.flags = flags self.num_running = 0 + # In case there is a graveyard of zombie bitcoinds, we can apply a + # pseudorandom offset to hopefully jump over them. + # (625 is PORT_RANGE/MAX_NODES) + self.portseed_offset = int(time.time() * 1000) % 625 self.jobs = [] def get_next(self): @@ -430,7 +434,9 @@ def get_next(self): # Add tests self.num_running += 1 t = self.test_list.pop(0) - port_seed = ["--portseed=%s" % len(self.test_list)] + port_seed = ["--portseed={}".format(len(self.test_list) + self.portseed_offset)] + log_stdout = tempfile.SpooledTemporaryFile(max_size=2**16) + log_stderr = tempfile.SpooledTemporaryFile(max_size=2**16) self.jobs.append((t, time.time(), subprocess.Popen((RPC_TESTS_DIR + t).split() + self.flags.split() + port_seed, From 8106a12c842f96fca54f7f96eda8a3b293d63624 Mon Sep 17 00:00:00 2001 From: Andrea Suisani Date: Fri, 19 May 2017 00:02:33 +0200 Subject: [PATCH 09/13] [qa] Print debug info on stdout rather than stderr excessive.py and validateblocktemplate.py use to print additional info on stderr. After we merged the machinery to run tests in parallel having a test writing on stderr means marking the test as failing even if successful. --- qa/rpc-tests/excessive.py | 2 +- qa/rpc-tests/validateblocktemplate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/excessive.py b/qa/rpc-tests/excessive.py index 8678fc19bb8..9ec02185606 100755 --- a/qa/rpc-tests/excessive.py +++ b/qa/rpc-tests/excessive.py @@ -18,7 +18,7 @@ if sys.version_info[0] < 3: raise "Use Python 3" import logging -logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO) +logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO, stream=sys.stdout) def mostly_sync_mempools(rpc_connections, difference=50, wait=1, verbose=1): diff --git a/qa/rpc-tests/validateblocktemplate.py b/qa/rpc-tests/validateblocktemplate.py index de2ef64429e..b2e46f7f0ba 100755 --- a/qa/rpc-tests/validateblocktemplate.py +++ b/qa/rpc-tests/validateblocktemplate.py @@ -10,7 +10,7 @@ if sys.version_info[0] < 3: raise "Use Python 3" import logging -logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO) +logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO, stream=sys.stdout) # Test validateblocktemplate RPC call from test_framework.key import CECKey From a761f3231afab54c151bff038c5128cf403078b5 Mon Sep 17 00:00:00 2001 From: Andrea Suisani Date: Fri, 19 May 2017 00:39:33 +0200 Subject: [PATCH 10/13] [qa] Adapt BU final test summary to the parallel execution Commit 758562c (parallel execution) introduce also a new way to summarize test results. With this commit we removed the old style report modulo number of passed/failed/skipped/disabled tests. --- qa/pull-tester/rpc-tests.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 4a1b7debb61..9185d6301f7 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -261,9 +261,7 @@ def show_wrapper_options(): def runtests(): global passOn coverage = None - execution_time = {} - test_passed = {} - test_failure_info = {} + test_passed = [] disabled = [] skipped = [] tests_to_run = [] @@ -367,6 +365,7 @@ def runtests(): for _ in range(len(tests_to_run)): (name, stdout, stderr, passed, duration) = job_queue.get_next() + test_passed.append(passed) all_passed = all_passed and passed time_sum += duration @@ -389,20 +388,8 @@ def runtests(): if not showHelp: # show some overall results and aggregates print() - print("%-50s Status Time (s)" % "Test") - print('-' * 70) - for k in sorted(execution_time.keys()): - print("%-50s %-6s %7s" % (k, "PASS" if test_passed[k] else "FAILED", execution_time[k])) - for d in disabled: - print("%-50s %-8s" % (d, "DISABLED")) - for s in skipped: - print("%-50s %-8s" % (s, "SKIPPED")) - print('-' * 70) - print("%-44s Total time (s): %7s" % (" ", sum(execution_time.values()))) - - print - print("%d test(s) passed / %d test(s) failed / %d test(s) executed" % (list(test_passed.values()).count(True), - list(test_passed.values()).count(False), + print("%d test(s) passed / %d test(s) failed / %d test(s) executed" % (test_passed.count(True), + test_passed.count(False), len(test_passed))) print("%d test(s) disabled / %d test(s) skipped due to platform" % (len(disabled), len(skipped))) From a42ac88bbcb919b07a38a1c59d9406ed797818b3 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 19 May 2017 01:20:02 +0200 Subject: [PATCH 11/13] [qa] pull-tester: Fix assertion and check for run_parallel (manual cherry-pick) --- qa/pull-tester/rpc-tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 9185d6301f7..9447d7736f8 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -351,7 +351,7 @@ def runtests(): trimmed_tests_to_run.append(t) tests_to_run = trimmed_tests_to_run - if len(tests_to_run) > 1: + if len(tests_to_run) > 1 and run_parallel: # Populate cache subprocess.check_output([RPC_TESTS_DIR + 'create_cache.py'] + [flags]) @@ -431,7 +431,7 @@ def get_next(self): stdout=subprocess.PIPE, stderr=subprocess.PIPE))) if not self.jobs: - raise IndexError('%s from empty list' % __name__) + raise IndexError('pop from empty list') while True: # Return first proc that finishes time.sleep(.5) From 5b2ae4266c15654bebd5a58bb738b51f736f44a7 Mon Sep 17 00:00:00 2001 From: Andrea Suisani Date: Wed, 16 Aug 2017 14:19:11 +0200 Subject: [PATCH 12/13] [qa] Print debug info on stdout rather than stderr This is the same as `8106a12` applied to `wallet.py` and `buip055.py` --- qa/rpc-tests/buip055.py | 2 +- qa/rpc-tests/wallet.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/buip055.py b/qa/rpc-tests/buip055.py index 1634f699504..73211704589 100755 --- a/qa/rpc-tests/buip055.py +++ b/qa/rpc-tests/buip055.py @@ -20,7 +20,7 @@ if sys.version_info[0] < 3: raise "Use Python 3" import logging -logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO) +logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO, stream=sys.stdout) NODE_BITCOIN_CASH = (1 << 5) invalidOpReturn = hexlify(b'Bitcoin: A Peer-to-Peer Electronic Cash System') diff --git a/qa/rpc-tests/wallet.py b/qa/rpc-tests/wallet.py index e7eb4b982c1..9efa8d58101 100755 --- a/qa/rpc-tests/wallet.py +++ b/qa/rpc-tests/wallet.py @@ -8,7 +8,7 @@ if sys.version_info[0] < 3: raise "Use Python 3" import logging -logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO) +logging.basicConfig(format='%(asctime)s.%(levelname)s: %(message)s', level=logging.INFO,stream=sys.stdout) from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * From c7069e6bc55e75b8a101a3e3dfa490d591f3624b Mon Sep 17 00:00:00 2001 From: Andrea Suisani Date: Wed, 16 Aug 2017 14:34:23 +0200 Subject: [PATCH 13/13] Use `os.path.join` rather than string concatenation This would guarantee to generate files path that are valid in every kind of OS you are going to run the tests. Unfortunately it seems there're a lot of other places where string concatenation is used. I think we should fix all of them if we want the tests suite to be portable. --- qa/rpc-tests/test_framework/test_framework.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/test_framework/test_framework.py b/qa/rpc-tests/test_framework/test_framework.py index 594d4571bce..9f2e943ca98 100755 --- a/qa/rpc-tests/test_framework/test_framework.py +++ b/qa/rpc-tests/test_framework/test_framework.py @@ -161,7 +161,7 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): random.seed(self.randomseed) print("Random seed: %s" % self.randomseed) - self.options.tmpdir += '/' + str(self.options.port_seed) + self.options.tmpdir = os.path.join(self.options.tmpdir, str(self.options.port_seed)) if self.options.trace_rpc: logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) @@ -171,7 +171,7 @@ def main(self,argsOverride=None,bitcoinConfDict=None,wallets=None): PortSeed.n = self.options.port_seed - os.environ['PATH'] = self.options.srcdir+":"+self.options.srcdir+"/qt:"+os.environ['PATH'] + os.environ['PATH'] = self.options.srcdir + ":" + os.path.join(self.options.srcdir, "qt") + ":" + os.environ['PATH'] check_json_precision()