os-autoinst · mergify · Jul 22, 2020 · Jul 20, 2020 · Jul 20, 2020 · Jul 20, 2020
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -177,3 +177,14 @@ install(
 )
 add_custom_target(install-openvswitch
     COMMAND "${CMAKE_COMMAND}" -DCMAKE_INSTALL_COMPONENT=openvswitch -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
+
+# add tests
+include(test-targets)
+
+# add target to update files containing dependency information
+add_custom_target(
+    update-deps
+    COMMENT "Updating dependency information"
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/update-deps" --specfile dist/rpm/os-autoinst.spec --dockerfile docker/travis_test/Dockerfile
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+)
diff --git a/Makefile.am b/Makefile.am
@@ -205,12 +205,11 @@ check-local-files-specified:
 
 test-yaml:
 	@which yamllint >/dev/null 2>&1 || echo "Command 'yamllint' not found, can not execute YAML syntax checks"
-	@# Fall back to find if there is no git, e.g. in package builds
-	yamllint --strict $$(git ls-files "*.yml" "*.yaml" 2>/dev/null || find -name '*.y*ml')
+	$(srcdir)/tools/check-yaml-syntax
 
 check-local: check-local-files-specified test-yaml
 	$(srcdir)/tools/tidy --check
-	PERL5LIB=tools/lib/perlcritic:$$PERL5LIB perlcritic --gentle --include Perl::Critic::Policy::HashKeyQuote $(srcdir)
+	$(srcdir)/tools/check-perl-style perlcritic $(srcdir)
 	test x$(CHECK_DOC) = x0 || $(MAKE) check-doc
 
 .PHONY: check-doc
@@ -227,26 +226,17 @@ clean-local:
 	-rm -rf *.tar.*
 	-rm -rf cover_db/
 
-# TESTS: Specify individual test files in a space separated lists to overwrite
-# recursive search
-TESTS ?= -r
-PATHRE = ^|$(PWD)/|\.\./
-COVER_OPTS = \
-	PERL5OPT="-MDevel::Cover=-db,$(abs_builddir)/cover_db,-select,($(PATHRE))(OpenQA|backend|consoles|ppmclibs)/|($(PATHRE))isotovideo|($(PATHRE))[^/]+\.pm,-ignore,\.t|data/tests/|fake/tests/|/usr/bin/prove,-coverage,statement"
-TEST_OPTS = \
-	PERL5LIB="$(PWD):$(PWD)/ppmclibs:$(PWD)/ppmclibs/blib/arch/auto/tinycv:$$PERL5LIB"
-
 .PHONY: test
 test:
-	( cd t && $(TEST_OPTS) prove $(TESTS) )
+	( $(srcdir)/tools/invoke-tests --build-directory $(abs_builddir) )
 
 .PHONY: testv
 testv:
-	( cd t && $(TEST_OPTS) prove -v $(TESTS) )
+	( PROVE_ARGS=-v $(srcdir)/tools/invoke-tests --build-directory $(abs_builddir) )
 
 .PHONY: test-cover
 test-cover:
-	( cd t && $(TEST_OPTS) $(COVER_OPTS) prove $(TESTS) )
+	( $(srcdir)/tools/invoke-tests --coverage --build-directory $(abs_builddir) )
 
 .PHONY: test-cover-summary
 test-cover-summary:

diff --git a/README.asciidoc b/README.asciidoc
@@ -73,34 +73,131 @@ http://os-autoinst.github.io/openQA/contact/, too.
 
 Issues are tracked on https://progress.opensuse.org/projects/openqav3/
 
-For an overview of the architecture, see link:architecture.md[doc/architecture.md].
+For an overview of the architecture, see link:doc/architecture.md[doc/architecture.md].
 
 Rules for commits
 ^^^^^^^^^^^^^^^^^
 
 * Every commit is checked by https://travis-ci.org/travis[Travis CI] as soon as
-you create a pull request but you *should* run the os-autoinst tests locally,
-i.e. call
+you create a pull request but you *should* run the os-autoinst tests locally. Checkout
+the build instructions for further details.
+
+* For git commit messages use the rules stated on
+http://chris.beams.io/posts/git-commit/[How to Write a Git Commit Message] as
+a reference
+
+* Every pull request is reviewed in a peer review to give feedback on possible
+implications and how we can help each other to improve
+
+If this is too much hassle for you feel free to provide incomplete pull
+requests for consideration or create an issue with a code change proposal.
+
+Build instructions
+------------------
+
+The required dependencies are delcared in `dependencies.yaml`. (The names listed within
+that file are specific to openSUSE but can be easily transferred to other distributions.)
+
+CMake
+^^^^^
+
+Create a build directory outside of the source directory. The following commands need
+to be invoked within that directory.
+
+Configure build:
+----
+cmake $path_to_os_autoinst_checkout
+----
+
+You can specify any of the standard CMake variables, e.g. `-DCMAKE_BUILD_TYPE=Debug`
+and `-DCMAKE_INSTALL_PREFIX=/custom/install/prefix`.
+
+The following examples assume that GNU Make is used. It is possible to generate for
+a different build tool by adding e.g. `-G Ninja` to the CMake arguments.
+
+Build executables and libraries:
+----
+make symlinks
+----
+
+This target also creates symlinks of the built executables and libraries within the
+source directory so `isotovideo` can find them.
+
+Run all tests:
+----
+make test
+----
+
+Run all Perl tests (`*.t` files found within the `t` directory):
+----
+make test-perl-testsuite
+----
+
+Run individual tests by specifying them explicitly:
+----
+make test-perl-testsuite TESTS="15-logging.t 28-signalblocker.t"
+----
+
+By default CTest is invoked in verbose mode because prove already provides condensed
+output. Add `-DVERBOSE_CTEST=OFF` to the CMake arguments to avoid that.
+
+Add additional arguments to the `prove` invocation, e.g. enable verbose output:
+----
+make test-perl-testsuite PROVE_ARGS=-v
+----
+
+Gather coverage data while running tests:
+----
+make test-perl-testsuite WITH_COVER_OPTIONS=1
+----
+
+Generate a coverage report from the gathered coverage data:
+----
+make coverage
+----
+
+If no coverage data has been gathered so far the `coverage` target will invoke the
+testsuite automatically.
+
+Reset gathered coverage data:
+----
+make coverage-reset
+----
+
+Install files for packaging:
+----
+make install DESTDIR=…
+----
+
+Further notes:
+
+* It is also possible to run `ctest` within the build directory directly instead of
+  using the mentioned targets.
+* All mentioned variables to influence the test execution (`TESTS`, `WITH_COVER_OPTIONS`, …)
+  can be combined and can also be used with the `coverage` target.
+
+GNU Autotools
+^^^^^^^^^^^^^
+
+The following commands need to be invoked within the top-level of the repository
+checkout.
+
+Configure build:
 ----
 ./autogen.sh
+----
+
+Build executables and libraries:
+----
 make
 ----
-once to setup your workspace and before every commit
+
+Run all tests:
 ----
 make check
 ----
 
-* You can also run individual tests by specifying them explicitly:
+Run individual tests by specifying them explicitly:
 ----
 make check TESTS=23-baseclass.t
 ----
-
-* For git commit messages use the rules stated on
-http://chris.beams.io/posts/git-commit/[How to Write a Git Commit Message] as
-a reference
-
-* Every pull request is reviewed in a peer review to give feedback on possible
-implications and how we can help each other to improve
-
-If this is too much hassle for you feel free to provide incomplete pull
-requests for consideration or create an issue with a code change proposal.
diff --git a/cmake/test-targets.cmake b/cmake/test-targets.cmake
@@ -0,0 +1,119 @@
+cmake_minimum_required(VERSION 3.3.0)
+
+enable_testing()
+
+# enable verbose CTest output by default
+# note: We're mainly using prove which already provides a condensed output by default. To be able
+#       to follow the prove output as usual and configure the test verbosity on prove-level it makes
+#       sense to configure CTest to be verbose by default.
+option(VERBOSE_CTEST "enables verbose tests on CTest level" ON)
+if (VERBOSE_CTEST)
+    set(CMAKE_CTEST_COMMAND ${CMAKE_CTEST_COMMAND} -V)
+endif ()
+
+# test for install target
+add_test(
+    NAME test-installed-files
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-installed-files" "${CMAKE_MAKE_PROGRAM}"
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+)
+
+# add test for YAML syntax
+find_program(YAMLLINT_PATH yamllint)
+if (YAMLLINT_PATH)
+    add_test(
+        NAME test-local-yaml-syntax
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-yaml-syntax" "${YAMLLINT_PATH}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+else ()
+    message(STATUS "Set YAMLLINT_PATH to the path of the yamllint executable to enable YAML syntax checks.")
+endif ()
+
+# add tidy check
+add_test(
+    NAME test-local-tidy
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/tidy" --check
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+)
+
+# add test for Perl syntax/style issues
+find_program(PERLCRITIC_PATH perlcritic)
+if (PERLCRITIC_PATH)
+    add_test(
+        NAME test-local-perl-style
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-perl-style" "${PERLCRITIC_PATH}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+else ()
+    message(STATUS "Set PERLCRITIC_PATH to the path of the perlcritic executable to enable Perl syntax/style checks.")
+endif ()
+
+# add spell checking for test API documentation
+find_program(PODSPELL_PATH podspell)
+find_program(SPELL_PATH spell)
+if (PODSPELL_PATH AND SPELL_PATH)
+    add_test(
+        NAME test-doc-testapi-spellchecking
+        COMMAND sh -c "\"${PODSPELL_PATH}\" \"${CMAKE_CURRENT_SOURCE_DIR}/testapi.pm\" | \"${SPELL_PATH}\""
+    )
+else ()
+    message(STATUS "Set PODSPELL_PATH/SPELL_PATH to the path of the podspell/spell executable to enable spell checking.")
+endif ()
+
+# add targets for invoking Perl test suite
+find_program(PROVE_PATH prove)
+if (PROVE_PATH)
+    set(INVOKE_TEST_ARGS --prove-tool "${PROVE_PATH}" --make-tool "${CMAKE_MAKE_PROGRAM}" --build-directory "${CMAKE_CURRENT_BINARY_DIR}")
+    add_test(
+        NAME test-perl-testsuite
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/invoke-tests" ${INVOKE_TEST_ARGS}
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+else ()
+    message(STATUS "Set PROVE_PATH to the path of the prove executable to enable running the Perl testsuite.")
+endif ()
+
+# add build system targets for invoking specific tests
+add_custom_target(test-local COMMAND ${CMAKE_CTEST_COMMAND} -R "test-local-.*" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+add_custom_target(test-doc COMMAND ${CMAKE_CTEST_COMMAND} -R "test-doc-.*" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+add_custom_target(test-installed-files COMMAND ${CMAKE_CTEST_COMMAND} -R "test-installed-files" WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_custom_target(test-perl-testsuite COMMAND ${CMAKE_CTEST_COMMAND} -R "test-perl-testsuite" WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_dependencies(test-perl-testsuite symlinks)
+
+# add target for computing test coverage of Perl test suite
+find_program(COVER_PATH cover)
+if (COVER_PATH AND PROVE_PATH)
+    add_custom_command(
+        COMMENT "Run Perl testsuite with coverage instrumentation"
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/invoke-tests" --coverage ${INVOKE_TEST_ARGS}
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/cover_db/structure"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+    add_custom_command(
+        COMMENT "Generate coverage report (HTML)"
+        COMMAND "${COVER_PATH}" -report html_basic "${CMAKE_CURRENT_BINARY_DIR}/cover_db"
+        DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/cover_db/structure"
+        OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/coverage.html"
+    )
+    add_custom_target(
+        coverage-reset
+        COMMENT "Resetting previously gathered Perl test suite coverage"
+        COMMAND rm -r "${CMAKE_CURRENT_BINARY_DIR}/cover_db"
+    )
+    add_custom_target(
+        coverage
+        COMMENT "Perl test suite coverage (HTML)"
+        DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/coverage.html"
+    )
+    add_dependencies(coverage symlinks)
+    add_custom_target(
+        coverage-codecov
+        COMMENT "Perl test suite coverage (codecov)"
+        COMMAND "${COVER_PATH}" -report codecov "${CMAKE_CURRENT_BINARY_DIR}/cover_db"
+        DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/cover_db/structure"
+    )
+    add_dependencies(coverage-codecov symlinks)
+else ()
+    message(STATUS "Set COVER_PATH to the path of the cover executable to enable coverage computition of the Perl testsuite.")
+endif ()
diff --git a/t/27-make-update-deps.t b/t/27-make-update-deps.t
@@ -26,16 +26,17 @@ if (not -e "$Bin/../.git") {
     exit;
 }
 
-chdir "$Bin/..";
-my $make = "make update-deps";
-my @out  = qx{$make};
-my $rc   = $?;
-die "Could not run $make: rc=$rc" if $rc;
+my $build_dir = $ENV{OS_AUTOINST_BUILD_DIRECTORY} || "$Bin/..";
+my $make_tool = $ENV{OS_AUTOINST_MAKE_TOOL}       || 'make';
+my $make_cmd  = "$make_tool update-deps";
 
-my @status = grep { not m/^\?/ } qx{git status --porcelain};
+chdir $build_dir;
+my @out = qx{$make_cmd};
+my $rc  = $?;
+die "Could not run $make_cmd: rc=$rc" if $rc;
 
-ok(!@status, "No changed files after '$make'")
-  or diag @status;
+my @status = grep { not m/^\?/ } qx{git -C "$Bin/.." status --porcelain};
+ok(!@status, "No changed files after '$make_cmd'") or diag @status;
 
 done_testing;
 
diff --git a/tools/check-installed-files b/tools/check-installed-files
@@ -1,8 +1,9 @@
 #!/bin/sh -e
 destdir="${destdir:-$(mktemp -d)}"
-make install DESTDIR="$destdir"
+checkoutdir="$(dirname "$0")/.."
+env DESTDIR="$destdir" "${1:-make}" install
 installed_files=$(find "$destdir" -name '*.pm')
-all_source_files=$(git ls-files "*.pm" 2>/dev/null || find "$(dirname "$0")/.." -name "*.pm" -printf "%P\n")
+all_source_files=$(git -C "$checkoutdir" ls-files "*.pm" 2>/dev/null || find "$checkoutdir" -name "*.pm" -printf "%P\n")
 source_files=$(echo "$all_source_files" | grep -v -E '(ppmclibs/|t/|tools/)')
 for i in $source_files; do
     if ! echo "$installed_files" | grep -q "$i"; then

diff --git a/tools/check-perl-style b/tools/check-perl-style
@@ -0,0 +1,3 @@
+#!/bin/sh -e
+export PERL5LIB=tools/lib/perlcritic:$PERL5LIB
+"${1:-perlcritic}" --gentle --include Perl::Critic::Policy::HashKeyQuote "${2:-$PWD}"
diff --git a/tools/check-yaml-syntax b/tools/check-yaml-syntax
@@ -0,0 +1,3 @@
+#!/bin/sh -e
+# fall back to find if there is no git, e.g. in package builds
+"${1:-yamllint}" --strict $(git ls-files "*.yml" "*.yaml" 2>/dev/null || find -name '*.y*ml')