CMake: Add targets for invoking tests

CMakeLists.txt

@@ -178,6 +178,9 @@ install(
 add_custom_target(install-openvswitch
     COMMAND "${CMAKE_COMMAND}" -DCMAKE_INSTALL_COMPONENT=openvswitch -P "${CMAKE_BINARY_DIR}/cmake_install.cmake")
 
+# add tests
+include(test-targets)
+
 # add target to update files containing dependency information
 add_custom_target(
     update-deps

Makefile.am

@@ -205,12 +205,11 @@ check-local-files-specified:
 
 test-yaml:
 	@which yamllint >/dev/null 2>&1 || echo "Command 'yamllint' not found, can not execute YAML syntax checks"
-	@# Fall back to find if there is no git, e.g. in package builds
-	yamllint --strict $$(git ls-files "*.yml" "*.yaml" 2>/dev/null || find -name '*.y*ml')
+	$(srcdir)/tools/check-yaml-syntax
 
 check-local: check-local-files-specified test-yaml
 	$(srcdir)/tools/tidy --check
-	PERL5LIB=tools/lib/perlcritic:$$PERL5LIB perlcritic --gentle --include Perl::Critic::Policy::HashKeyQuote $(srcdir)
+	$(srcdir)/tools/check-perl-style perlcritic $(srcdir)
 	test x$(CHECK_DOC) = x0 || $(MAKE) check-doc
 
 .PHONY: check-doc
@@ -227,64 +226,17 @@ clean-local:
 	-rm -rf *.tar.*
 	-rm -rf cover_db/
 
-# Prevent error "blank line following trailing backslash" when deleting last
-# line in list in spec file to exclude tests
-EMPTY =
-TESTS ?= \
-	00-compile-check-all.t \
-	01-test_needle.t \
-	02-test_ocr.t \
-	03-testapi.t \
-	04-check_vars_docu.t \
-	05-pod.t \
-	06-pod-coverage.t \
-	07-commands.t \
-	08-autotest.t \
-	09-lockapi.t \
-	10-terminal.t \
-	10-virtio_terminal.t \
-	10-test-image-conversion-benchmark.t \
-	11-image-ppm.t \
-	12-bmwqemu.t \
-	13-osutils.t \
-	14-isotovideo.t \
-	15-logging.t \
-	16-send_with_fd.t \
-	17-basetest.t \
-	18-qemu.t \
-	18-backend-qemu.t \
-	18-qemu-options.t \
-	19-isotovideo-command-processing.t \
-	20-openqa-benchmark-stopwatch-utils.t \
-	21-needle-downloader.t \
-	22-svirt.t \
-	23-baseclass.t \
-	24-myjsonrpc.t \
-	24-myjsonrpc-debug.t \
-	25-spvm.t \
-	26-serial_screen.t \
-	26-ssh_screen.t \
-	27-make-update-deps.t \
-	99-full-stack.t \
-	$(EMPTY)
-
-PATHRE = ^|$(PWD)/|\.\./
-COVER_OPTS = \
-	PERL5OPT="-MDevel::Cover=-db,$(abs_builddir)/cover_db,-select,($(PATHRE))(OpenQA|backend|consoles|ppmclibs)/|($(PATHRE))isotovideo|($(PATHRE))[^/]+\.pm,-ignore,\.t|data/tests/|fake/tests/|/usr/bin/prove,-coverage,statement"
-TEST_OPTS = \
-	PERL5LIB="$(PWD):$(PWD)/ppmclibs:$(PWD)/ppmclibs/blib/arch/auto/tinycv:$$PERL5LIB"
-
 .PHONY: test
 test:
-	( cd t && $(TEST_OPTS) prove $(TESTS) )
+	( $(srcdir)/tools/invoke-tests prove make $(abs_builddir) )
 
 .PHONY: testv
 testv:
-	( cd t && $(TEST_OPTS) prove -v $(TESTS) )
+	( PROVE_ARGS=-v $(srcdir)/tools/invoke-tests prove make $(abs_builddir) )
 
 .PHONY: test-cover
 test-cover:
-	( cd t && $(TEST_OPTS) $(COVER_OPTS) prove $(TESTS) )
+	( $(srcdir)/tools/invoke-tests --coverage prove make $(abs_builddir) )
 
 .PHONY: test-cover-summary
 test-cover-summary:

cmake/test-targets.cmake

@@ -0,0 +1,72 @@
+cmake_minimum_required(VERSION 3.3.0)
+
+enable_testing()
+
+# add test for install target
+add_test(
+    NAME test-installed-files
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-installed-files" "${CMAKE_MAKE_PROGRAM}"
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+)
+
+# add test for YAML syntax
+find_program(YAMLLINT_PATH yamllint)
+if (YAMLLINT_PATH)
+    add_test(
+        NAME test-local-yaml-syntax
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-yaml-syntax" "${YAMLLINT_PATH}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+else ()
+    message(STATUS "Set YAMLLINT_PATH to the path of the yamllint executable to enable YAML syntax checks.")
+endif ()
+
+# add tidy check
+add_test(
+    NAME test-local-tidy
+    COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/tidy" --check
+    WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+)
+
+# add test for Perl syntax/style issues
+find_program(PERLCRITIC_PATH perlcritic)
+if (PERLCRITIC_PATH)
+    add_test(
+        NAME test-local-perl-style
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-perl-style" "${PERLCRITIC_PATH}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+else ()
+    message(STATUS "Set PERLCRITIC_PATH to the path of the perlcritic executable to enable Perl syntax/style checks.")
+endif ()
+
+# add spell checking for test API documentation
+find_program(PODSPELL_PATH podspell)
+find_program(SPELL_PATH spell)
+if (PODSPELL_PATH AND SPELL_PATH)
+    add_test(
+        NAME test-doc-testapi-spellchecking
+        COMMAND sh -c "\"${PODSPELL_PATH}\" \"${CMAKE_CURRENT_SOURCE_DIR}/testapi.pm\" | \"${SPELL_PATH}\""
+    )
+else ()
+    message(STATUS "Set PODSPELL_PATH/SPELL_PATH to the path of the podspell/spell executable to enable spell checking.")
+endif ()
+
+# add targets for invoking Perl test suite
+find_program(PROVE_PATH prove)
+if (PROVE_PATH)
+    add_test(
+        NAME test-perl-testsuite
+        COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/invoke-tests" "${PROVE_PATH}" "${CMAKE_MAKE_PROGRAM}" "${CMAKE_CURRENT_BINARY_DIR}"
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+    )
+else ()
+    message(STATUS "Set PROVE_PATH to the path of the prove executable to enable running the Perl testsuite.")
+endif ()
+
+# add build system targets for invoking specific tests
+add_custom_target(test-local COMMAND ${CMAKE_CTEST_COMMAND} -R "test-local-.*" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+add_custom_target(test-doc COMMAND ${CMAKE_CTEST_COMMAND} -R "test-doc-.*" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}")
+add_custom_target(test-installed-files COMMAND ${CMAKE_CTEST_COMMAND} -R "test-installed-files" WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_custom_target(test-perl-testsuite COMMAND ${CMAKE_CTEST_COMMAND} -R "test-perl-testsuite" WORKING_DIRECTORY ${CMAKE_BINARY_DIR})
+add_dependencies(test-perl-testsuite symlinks)

t/27-make-update-deps.t

@@ -26,16 +26,17 @@ if (not -e "$Bin/../.git") {
     exit;
 }
 
-chdir "$Bin/..";
-my $make = "make update-deps";
-my @out  = qx{$make};
-my $rc   = $?;
-die "Could not run $make: rc=$rc" if $rc;
+my $build_dir = $ENV{OS_AUTOINST_BUILD_DIRECTORY} || "$Bin/..";
+my $make_tool = $ENV{OS_AUTOINST_MAKE_TOOL}       || 'make';
+my $make_cmd  = "$make_tool update-deps";
 
-my @status = grep { not m/^\?/ } qx{git status --porcelain};
+chdir $build_dir;
+my @out = qx{$make_cmd};
+my $rc  = $?;
+die "Could not run $make_cmd: rc=$rc" if $rc;
 
-ok(!@status, "No changed files after '$make'")
-  or diag @status;
+my @status = grep { not m/^\?/ } qx{git -C "$Bin/.." status --porcelain};
+ok(!@status, "No changed files after '$make_cmd'") or diag @status;
 
 done_testing;
 

tools/check-installed-files

@@ -1,8 +1,9 @@
 #!/bin/sh -e
 destdir="${destdir:-$(mktemp -d)}"
-make install DESTDIR="$destdir"
+checkoutdir="$(dirname "$0")/.."
+env DESTDIR="$destdir" "${1:-make}" install
 installed_files=$(find "$destdir" -name '*.pm')
-all_source_files=$(git ls-files "*.pm" 2>/dev/null || find "$(dirname "$0")/.." -name "*.pm" -printf "%P\n")
+all_source_files=$(git -C "$checkoutdir" ls-files "*.pm" 2>/dev/null || find "$checkoutdir" -name "*.pm" -printf "%P\n")
 source_files=$(echo "$all_source_files" | grep -v -E '(ppmclibs/|t/|tools/)')
 for i in $source_files; do
     if ! echo "$installed_files" | grep -q "$i"; then

tools/check-perl-style

@@ -0,0 +1,3 @@
+#!/bin/sh -e
+export PERL5LIB=tools/lib/perlcritic:$PERL5LIB
+"${1:-perlcritic}" --gentle --include Perl::Critic::Policy::HashKeyQuote "${2:-$PWD}"

tools/check-yaml-syntax

@@ -0,0 +1,3 @@
+#!/bin/sh -e
+# fall back to find if there is no git, e.g. in package builds
+"${1:-yamllint}" --strict $(git ls-files "*.yml" "*.yaml" 2>/dev/null || find -name '*.y*ml')

tools/invoke-tests

@@ -0,0 +1,68 @@
+#!/bin/bash -e
+
+# process arguments
+if [[ $1 == --coverage ]]; then
+    WITH_COVER_OPTIONS=1
+    shift
+fi
+if [[ $# -lt 3 ]]; then
+    echo "usage: $0 [--coverage] prove-tool-path make-tool-path build-directory [test-file...]"
+    exit
+fi
+prove_path=$1 make_path=$2 build_dir=$3
+shift 3
+test_files=("$@")
+
+# set Perl module include path and coverage options
+export PERL5LIB="$PWD:$PWD/ppmclibs:$PWD/ppmclibs/blib/arch/auto/tinycv:$PERL5LIB"
+if [[ $WITH_COVER_OPTIONS ]]; then
+    path_regex='^|$(PWD)/|\.\./'
+    export PERL5OPT="-MDevel::Cover=-db,$build_dir/cover_db,-select,($path_regex)(OpenQA|backend|consoles|ppmclibs)/|($path_regex)isotovideo|($path_regex)[^/]+\.pm,-ignore,\.t|data/tests/|fake/tests/|$prove_path,-coverage,statement"
+fi
+
+# set variables for tests which need to invoke the make tool
+export OS_AUTOINST_BUILD_DIRECTORY=$build_dir
+export OS_AUTOINST_MAKE_TOOL=$make_path
+
+# define list of tests to run by default; allow specifying the list of tests via TESTS in consistency with openQA
+[[ ${#test_files[@]} -lt 1 ]] && test_files=($TESTS)
+[[ ${#test_files[@]} -lt 1 ]] && test_files=(
+    00-compile-check-all.t
+    01-test_needle.t
+    02-test_ocr.t
+    03-testapi.t
+    04-check_vars_docu.t
+    05-pod.t
+    06-pod-coverage.t
+    07-commands.t
+    08-autotest.t
+    09-lockapi.t
+    10-terminal.t
+    10-virtio_terminal.t
+    10-test-image-conversion-benchmark.t
+    11-image-ppm.t
+    12-bmwqemu.t
+    13-osutils.t
+    14-isotovideo.t
+    15-logging.t
+    16-send_with_fd.t
+    17-basetest.t
+    18-qemu.t
+    18-backend-qemu.t
+    18-qemu-options.t
+    19-isotovideo-command-processing.t
+    20-openqa-benchmark-stopwatch-utils.t
+    21-needle-downloader.t
+    22-svirt.t
+    23-baseclass.t
+    24-myjsonrpc.t
+    24-myjsonrpc-debug.t
+    25-spvm.t
+    26-serial_screen.t
+    26-ssh_screen.t
+    27-make-update-deps.t
+    99-full-stack.t
+)
+
+# invoke tests within the t directory
+cd t && "$prove_path" $PROVE_ARGS "${test_files[@]}"