Merge bitcoin#9956: Reorganise qa directory (#2912)

* Merge bitcoin#9956: Reorganise qa directory

63d66ba Move src/test/bitcoin-util-test.py to test/util/bitcoin-util-test.py (John Newbery)
5b0bff4 Rename --enable-extended-rpc-tests to --enable-extended-functional-tests (John Newbery)
a9bd622 Rename test/pull-tester/rpc-tests.py to test/functional/test_runner.py (John Newbery)
c28ee91 Rename rpc-tests directory to functional (John Newbery)
00902c4 Rename qa directory to test (John Newbery)

Tree-SHA512: ee7125c0c647d81590177beef2c8852c4ef76fdcf888096d9d4d360562a01d8d3b453345c3040487b2a043935bd1e7e80018f34462d6e02262bedbe23edcc576

resolve build errors

Signed-off-by: Pasta <Pasta@dash.org>

update test_runner.py in testintegrations.sh

Signed-off-by: Pasta <Pasta@dash.org>

* moved dash specific tests

Signed-off-by: Pasta <Pasta@dash.org>

* dashify README.md

Signed-off-by: Pasta <Pasta@dash.org>

* removed autoix*.py

Signed-off-by: Pasta <Pasta@dash.org>

* change back file perms

* dedashify

Signed-off-by: Pasta <Pasta@dash.org>

Author: PastaPastaPasta

.gitignore

@@ -108,9 +108,9 @@ coverage_percent.txt
 linux-coverage-build
 linux-build
 win32-build
-qa/pull-tester/tests_config.py
-qa/pull-tester/tests_config.ini
-qa/cache/*
+test/functional/config.ini
+test/util/buildenv.py
+test/cache/*
 
 !src/leveldb*/Makefile
 

Makefile.am

@@ -61,7 +61,7 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \
 
 COVERAGE_INFO = baseline_filtered_combined.info baseline.info \
   leveldb_baseline.info test_dash_filtered.info total_coverage.info \
-  baseline_filtered.info rpc_test.info rpc_test_filtered.info \
+  baseline_filtered.info functional_test.info functional_test_filtered.info \
   leveldb_baseline_filtered.info test_dash_coverage.info test_dash.info
 
 dist-hook:
@@ -191,20 +191,20 @@ test_dash.info: baseline_filtered_combined.info
 test_dash_filtered.info: test_dash.info
 	$(LCOV) -r $< "/usr/include/*" -o $@
 
-rpc_test.info: test_dash_filtered.info
-	-@TIMEOUT=15 python qa/pull-tester/rpc-tests.py $(EXTENDED_RPC_TESTS)
-	$(LCOV) -c -d $(abs_builddir)/src --t rpc-tests -o $@
+functional_test.info: test_dash_filtered.info
+	-@TIMEOUT=15 python test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS)
+	$(LCOV) -c -d $(abs_builddir)/src --t functional-tests -o $@
 	$(LCOV) -z -d $(abs_builddir)/src
 	$(LCOV) -z -d $(abs_builddir)/src/leveldb
 
-rpc_test_filtered.info: rpc_test.info
+functional_test_filtered.info: functional_test.info
 	$(LCOV) -r $< "/usr/include/*" -o $@
 
 test_dash_coverage.info: baseline_filtered_combined.info test_dash_filtered.info
 	$(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_dash_filtered.info -o $@
 
-total_coverage.info: baseline_filtered_combined.info test_dash_filtered.info rpc_test_filtered.info
-	$(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_dash_filtered.info -a rpc_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
+total_coverage.info: baseline_filtered_combined.info test_dash_filtered.info functional_test_filtered.info
+	$(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_dash_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
 
 test_dash.coverage/.dirstamp:  test_dash_coverage.info
 	$(GENHTML) -s $< -o $(@D)
@@ -220,14 +220,58 @@ endif
 
 dist_noinst_SCRIPTS = autogen.sh
 
-EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
+
+EXTRA_DIST += \
+    test/util/bctest.py \
+    test/util/bitcoin-util-test.py \
+    test/util/data/bitcoin-util-test.json \
+    test/util/data/blanktxv1.hex \
+    test/util/data/blanktxv1.json \
+    test/util/data/blanktxv2.hex \
+    test/util/data/blanktxv2.json \
+    test/util/data/tt-delin1-out.hex \
+    test/util/data/tt-delin1-out.json \
+    test/util/data/tt-delout1-out.hex \
+    test/util/data/tt-delout1-out.json \
+    test/util/data/tt-locktime317000-out.hex \
+    test/util/data/tt-locktime317000-out.json \
+    test/util/data/tx394b54bb.hex \
+    test/util/data/txcreate1.hex \
+    test/util/data/txcreate1.json \
+    test/util/data/txcreate2.hex \
+    test/util/data/txcreate2.json \
+    test/util/data/txcreatedata1.hex \
+    test/util/data/txcreatedata1.json \
+    test/util/data/txcreatedata2.hex \
+    test/util/data/txcreatedata2.json \
+    test/util/data/txcreatedata_seq0.hex \
+    test/util/data/txcreatedata_seq0.json \
+    test/util/data/txcreatedata_seq1.hex \
+    test/util/data/txcreatedata_seq1.json \
+    test/util/data/txcreatemultisig1.hex \
+    test/util/data/txcreatemultisig1.json \
+    test/util/data/txcreatemultisig2.hex \
+    test/util/data/txcreatemultisig2.json \
+    test/util/data/txcreateoutpubkey1.hex \
+    test/util/data/txcreateoutpubkey1.json \
+    test/util/data/txcreatescript1.hex \
+    test/util/data/txcreatescript1.json \
+    test/util/data/txcreatescript2.hex \
+    test/util/data/txcreatescript2.json \
+    test/util/data/txcreatesignv1.hex \
+    test/util/data/txcreatesignv1.json \
+    test/util/data/txcreatesignv2.hex
 
 CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER)
 
+# This file is problematic for out-of-tree builds if it exists.
+DISTCLEANFILES = test/util/buildenv.pyc
+
 .INTERMEDIATE: $(COVERAGE_INFO)
 
 DISTCHECK_CONFIGURE_FLAGS = --enable-man
 
 clean-local:
-	rm -rf coverage_percent.txt test_dash.coverage/ total.coverage/ qa/tmp/ cache/ $(OSX_APP)
-	rm -rf qa/pull-tester/__pycache__
+	rm -rf coverage_percent.txt test_dash.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP)
+	rm -rf test/functional/__pycache__

README.md

@@ -49,9 +49,9 @@ submit new unit tests for old code. Unit tests can be compiled and run
 (assuming they weren't disabled in configure) with: `make check`. Further details on running
 and extending unit tests can be found in [/src/test/README.md](/src/test/README.md).
 
-There are also [regression and integration tests](/qa) of the RPC interface, written
+There are also [regression and integration tests](/test), written
 in Python, that are run automatically on the build server.
-These tests can be run (if the [test dependencies](/qa) are installed) with: `qa/pull-tester/rpc-tests.py`
+These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py`
 
 The Travis CI system makes sure that every pull request is built for Windows, Linux, and OS X, and that unit/sanity tests are run automatically.
 

ci/test_integrationtests.sh

@@ -17,4 +17,4 @@ export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib
 
 cd build-ci/dashcore-$BUILD_TARGET
 
-./qa/pull-tester/rpc-tests.py --coverage $PASS_ARGS
+./test/functional/test_runner.py --coverage $PASS_ARGS

configure.ac

@@ -124,10 +124,10 @@ AC_ARG_ENABLE(bench,
     [use_bench=$enableval],
     [use_bench=yes])
 
-AC_ARG_ENABLE([extended-rpc-tests],
-    AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]),
-    [use_extended_rpc_tests=$enableval],
-    [use_extended_rpc_tests=no])
+AC_ARG_ENABLE([extended-functional-tests],
+    AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests when using lcov (default no)]),
+    [use_extended_functional_tests=$enableval],
+    [use_extended_functional_tests=no])
 
 AC_ARG_WITH([qrencode],
   [AS_HELP_STRING([--with-qrencode],
@@ -459,8 +459,8 @@ if test x$use_pkgconfig = xyes; then
   ])
 fi
 
-if test x$use_extended_rpc_tests != xno; then
-  AC_SUBST(EXTENDED_RPC_TESTS, -extended)
+if test x$use_extended_functional_tests != xno; then
+  AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended)
 fi
 
 if test x$use_lcov = xyes; then
@@ -1200,10 +1200,12 @@ AC_SUBST(EVENT_PTHREADS_LIBS)
 AC_SUBST(ZMQ_LIBS)
 AC_SUBST(PROTOBUF_LIBS)
 AC_SUBST(QR_LIBS)
-AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py])
-AC_CONFIG_FILES([qa/pull-tester/tests_config.ini],[chmod +x qa/pull-tester/tests_config.ini])
+AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini])
+AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py])
 AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
-AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py])
+AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])
+AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py])
+AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py])
 
 dnl boost's m4 checks do something really nasty: they export these vars. As a
 dnl result, they leak into secp256k1's configure and crazy things happen.
@@ -1251,8 +1253,8 @@ esac
 dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows
 case ${OS} in
    *Windows*)
-     sed  's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/'  qa/pull-tester/tests_config.py > qa/pull-tester/tests_config-2.py
-     mv qa/pull-tester/tests_config-2.py qa/pull-tester/tests_config.py
+     sed  's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/'  test/functional/config.ini > test/functional/config-2.ini
+     mv test/functional/config-2.ini test/functional/config.ini
    ;;
 esac
 

contrib/devtools/copyright_header.py

@@ -33,7 +33,7 @@
     'src/tinyformat.h',
     'src/leveldb/util/env_win.cc',
     'src/crypto/ctaes/bench.c',
-    'qa/rpc-tests/test_framework/bignum.py',
+    'test/functional/test_framework/bignum.py',
     # python init:
     '*__init__.py',
 ]

doc/developer-notes.md

@@ -133,7 +133,7 @@ Run with the -testnet option to run with "play coins" on the test network, if yo
 are testing multi-machine code that needs to operate across the internet.
 
 If you are testing something that can run on one machine, run with the -regtest option.
-In regression test mode, blocks can be created on-demand; see qa/rpc-tests/ for tests
+In regression test mode, blocks can be created on-demand; see test/functional/ for tests
 that run in -regtest mode.
 
 **DEBUG_LOCKORDER**
@@ -255,7 +255,7 @@ Wallet
 
   - *Rationale*: In RPC code that conditionally uses the wallet (such as
     `validateaddress`) it is easy to forget that global pointer `pwalletMain`
-    can be NULL. See `qa/rpc-tests/disablewallet.py` for functional tests
+    can be NULL. See `test/functional/disablewallet.py` for functional tests
     exercising the API with `-disablewallet`
 
 - Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set

qa/README.md

@@ -8,48 +8,6 @@ bin_PROGRAMS += test/test_dash
 TEST_SRCDIR = test
 TEST_BINARY=test/test_dash$(EXEEXT)
 
-
-EXTRA_DIST += \
-	test/bctest.py \
-	test/bitcoin-util-test.py \
-	test/data/bitcoin-util-test.json \
-	test/data/blanktxv1.hex \
-	test/data/blanktxv1.json \
-	test/data/blanktxv2.hex \
-	test/data/blanktxv2.json \
-	test/data/tt-delin1-out.hex \
-	test/data/tt-delin1-out.json \
-	test/data/tt-delout1-out.hex \
-	test/data/tt-delout1-out.json \
-	test/data/tt-locktime317000-out.hex \
-	test/data/tt-locktime317000-out.json \
-	test/data/tx394b54bb.hex \
-	test/data/txcreate1.hex \
-	test/data/txcreate1.json \
-	test/data/txcreate2.hex \
-	test/data/txcreate2.json \
-	test/data/txcreatedata1.hex \
-	test/data/txcreatedata1.json \
-	test/data/txcreatedata2.hex \
-	test/data/txcreatedata2.json \
-	test/data/txcreatedata_seq0.hex \
-	test/data/txcreatedata_seq0.json \
-	test/data/txcreatedata_seq1.hex \
-	test/data/txcreatedata_seq1.json \
-	test/data/txcreatemultisig1.hex \
-	test/data/txcreatemultisig1.json \
-	test/data/txcreatemultisig2.hex \
-	test/data/txcreatemultisig2.json \
-	test/data/txcreateoutpubkey1.hex \
-	test/data/txcreateoutpubkey1.json \
-	test/data/txcreatescript1.hex \
-	test/data/txcreatescript1.json \
-	test/data/txcreatescript2.hex \
-	test/data/txcreatescript2.json \
-	test/data/txcreatesignv1.hex \
-	test/data/txcreatesignv1.json \
-	test/data/txcreatesignv2.hex
-
 JSON_TEST_FILES = \
   test/data/script_tests.json \
   test/data/base58_keys_valid.json \
@@ -174,9 +132,6 @@ CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno $(GENERATED_TEST_FILES)
 
 CLEANFILES += $(CLEAN_BITCOIN_TEST)
 
-# This file is problematic for out-of-tree builds if it exists.
-DISTCLEANFILES += test/buildenv.pyc
-
 dash_test: $(TEST_BINARY)
 
 dash_test_check: $(TEST_BINARY) FORCE
@@ -186,8 +141,8 @@ dash_test_clean : FORCE
 	rm -f $(CLEAN_BITCOIN_TEST) $(test_test_dash_OBJECTS) $(TEST_BINARY)
 
 check-local:
-	@echo "Running test/bitcoin-util-test.py..."
-	$(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(PYTHON) $(srcdir)/test/bitcoin-util-test.py
+	@echo "Running test/util/bitcoin-util-test.py..."
+	$(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check
 if EMBEDDED_UNIVALUE
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check

src/Makefile.test.include

@@ -39,7 +39,7 @@ unit tests as possible).
 
 The build system is setup to compile an executable called `test_dash`
 that runs all of the unit tests.  The main source file is called
-test_dash.cpp. To add a new unit test file to our test suite you need 
+test_dash.cpp. To add a new unit test file to our test suite you need
 to add the file to `src/Makefile.test.include`. The pattern is to create 
 one test file for each class or source file for which you want to create 
 unit tests.  The file naming convention is `<source_filename>_tests.cpp` 
@@ -50,12 +50,3 @@ examine `uint256_tests.cpp`.
 For further reading, I found the following website to be helpful in
 explaining how the boost unit test framework works:
 [http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/).
-
-### bitcoin-util-test.py
-
-The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just dash-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory:
-
-```
-test/bitcoin-util-test.py --srcdir=[current directory]
-
-```