diff --git a/CMakeLists.txt b/CMakeLists.txt index b6491293fde..8e347cc2cfa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -177,3 +177,14 @@ install( ) add_custom_target(install-openvswitch COMMAND "${CMAKE_COMMAND}" -DCMAKE_INSTALL_COMPONENT=openvswitch -P "${CMAKE_BINARY_DIR}/cmake_install.cmake") + +# add tests +include(test-targets) + +# add target to update files containing dependency information +add_custom_target( + update-deps + COMMENT "Updating dependency information" + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/update-deps" --specfile dist/rpm/os-autoinst.spec --dockerfile docker/travis_test/Dockerfile + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" +) diff --git a/Makefile.am b/Makefile.am index 9a172d4ac72..856eddfabff 100644 --- a/Makefile.am +++ b/Makefile.am @@ -205,12 +205,11 @@ check-local-files-specified: test-yaml: @which yamllint >/dev/null 2>&1 || echo "Command 'yamllint' not found, can not execute YAML syntax checks" - @# Fall back to find if there is no git, e.g. in package builds - yamllint --strict $$(git ls-files "*.yml" "*.yaml" 2>/dev/null || find -name '*.y*ml') + $(srcdir)/tools/check-yaml-syntax check-local: check-local-files-specified test-yaml $(srcdir)/tools/tidy --check - PERL5LIB=tools/lib/perlcritic:$$PERL5LIB perlcritic --gentle --include Perl::Critic::Policy::HashKeyQuote $(srcdir) + $(srcdir)/tools/check-perl-style perlcritic $(srcdir) test x$(CHECK_DOC) = x0 || $(MAKE) check-doc .PHONY: check-doc @@ -227,26 +226,17 @@ clean-local: -rm -rf *.tar.* -rm -rf cover_db/ -# TESTS: Specify individual test files in a space separated lists to overwrite -# recursive search -TESTS ?= -r -PATHRE = ^|$(PWD)/|\.\./ -COVER_OPTS = \ - PERL5OPT="-MDevel::Cover=-db,$(abs_builddir)/cover_db,-select,($(PATHRE))(OpenQA|backend|consoles|ppmclibs)/|($(PATHRE))isotovideo|($(PATHRE))[^/]+\.pm,-ignore,\.t|data/tests/|fake/tests/|/usr/bin/prove,-coverage,statement" -TEST_OPTS = \ - PERL5LIB="$(PWD):$(PWD)/ppmclibs:$(PWD)/ppmclibs/blib/arch/auto/tinycv:$$PERL5LIB" - .PHONY: test test: - ( cd t && $(TEST_OPTS) prove $(TESTS) ) + ( $(srcdir)/tools/invoke-tests --build-directory $(abs_builddir) ) .PHONY: testv testv: - ( cd t && $(TEST_OPTS) prove -v $(TESTS) ) + ( PROVE_ARGS=-v $(srcdir)/tools/invoke-tests --build-directory $(abs_builddir) ) .PHONY: test-cover test-cover: - ( cd t && $(TEST_OPTS) $(COVER_OPTS) prove $(TESTS) ) + ( $(srcdir)/tools/invoke-tests --coverage --build-directory $(abs_builddir) ) .PHONY: test-cover-summary test-cover-summary: diff --git a/README.asciidoc b/README.asciidoc index 3c4d1bee30f..bde6271628d 100644 --- a/README.asciidoc +++ b/README.asciidoc @@ -73,34 +73,131 @@ http://os-autoinst.github.io/openQA/contact/, too. Issues are tracked on https://progress.opensuse.org/projects/openqav3/ -For an overview of the architecture, see link:architecture.md[doc/architecture.md]. +For an overview of the architecture, see link:doc/architecture.md[doc/architecture.md]. Rules for commits ^^^^^^^^^^^^^^^^^ * Every commit is checked by https://travis-ci.org/travis[Travis CI] as soon as -you create a pull request but you *should* run the os-autoinst tests locally, -i.e. call +you create a pull request but you *should* run the os-autoinst tests locally. Checkout +the build instructions for further details. + +* For git commit messages use the rules stated on +http://chris.beams.io/posts/git-commit/[How to Write a Git Commit Message] as +a reference + +* Every pull request is reviewed in a peer review to give feedback on possible +implications and how we can help each other to improve + +If this is too much hassle for you feel free to provide incomplete pull +requests for consideration or create an issue with a code change proposal. + +Build instructions +------------------ + +The required dependencies are delcared in `dependencies.yaml`. (The names listed within +that file are specific to openSUSE but can be easily transferred to other distributions.) + +CMake +^^^^^ + +Create a build directory outside of the source directory. The following commands need +to be invoked within that directory. + +Configure build: +---- +cmake $path_to_os_autoinst_checkout +---- + +You can specify any of the standard CMake variables, e.g. `-DCMAKE_BUILD_TYPE=Debug` +and `-DCMAKE_INSTALL_PREFIX=/custom/install/prefix`. + +The following examples assume that GNU Make is used. It is possible to generate for +a different build tool by adding e.g. `-G Ninja` to the CMake arguments. + +Build executables and libraries: +---- +make symlinks +---- + +This target also creates symlinks of the built executables and libraries within the +source directory so `isotovideo` can find them. + +Run all tests: +---- +make test +---- + +Run all Perl tests (`*.t` files found within the `t` directory): +---- +make test-perl-testsuite +---- + +Run individual tests by specifying them explicitly: +---- +make test-perl-testsuite TESTS="15-logging.t 28-signalblocker.t" +---- + +By default CTest is invoked in verbose mode because prove already provides condensed +output. Add `-DVERBOSE_CTEST=OFF` to the CMake arguments to avoid that. + +Add additional arguments to the `prove` invocation, e.g. enable verbose output: +---- +make test-perl-testsuite PROVE_ARGS=-v +---- + +Gather coverage data while running tests: +---- +make test-perl-testsuite WITH_COVER_OPTIONS=1 +---- + +Generate a coverage report from the gathered coverage data: +---- +make coverage +---- + +If no coverage data has been gathered so far the `coverage` target will invoke the +testsuite automatically. + +Reset gathered coverage data: +---- +make coverage-reset +---- + +Install files for packaging: +---- +make install DESTDIR=… +---- + +Further notes: + +* It is also possible to run `ctest` within the build directory directly instead of + using the mentioned targets. +* All mentioned variables to influence the test execution (`TESTS`, `WITH_COVER_OPTIONS`, …) + can be combined and can also be used with the `coverage` target. + +GNU Autotools +^^^^^^^^^^^^^ + +The following commands need to be invoked within the top-level of the repository +checkout. + +Configure build: ---- ./autogen.sh +---- + +Build executables and libraries: +---- make ---- -once to setup your workspace and before every commit + +Run all tests: ---- make check ---- -* You can also run individual tests by specifying them explicitly: +Run individual tests by specifying them explicitly: ---- make check TESTS=23-baseclass.t ---- - -* For git commit messages use the rules stated on -http://chris.beams.io/posts/git-commit/[How to Write a Git Commit Message] as -a reference - -* Every pull request is reviewed in a peer review to give feedback on possible -implications and how we can help each other to improve - -If this is too much hassle for you feel free to provide incomplete pull -requests for consideration or create an issue with a code change proposal. diff --git a/cmake/test-targets.cmake b/cmake/test-targets.cmake new file mode 100644 index 00000000000..79ce9220644 --- /dev/null +++ b/cmake/test-targets.cmake @@ -0,0 +1,119 @@ +cmake_minimum_required(VERSION 3.3.0) + +enable_testing() + +# enable verbose CTest output by default +# note: We're mainly using prove which already provides a condensed output by default. To be able +# to follow the prove output as usual and configure the test verbosity on prove-level it makes +# sense to configure CTest to be verbose by default. +option(VERBOSE_CTEST "enables verbose tests on CTest level" ON) +if (VERBOSE_CTEST) + set(CMAKE_CTEST_COMMAND ${CMAKE_CTEST_COMMAND} -V) +endif () + +# test for install target +add_test( + NAME test-installed-files + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-installed-files" "${CMAKE_MAKE_PROGRAM}" + WORKING_DIRECTORY "${CMAKE_BINARY_DIR}" +) + +# add test for YAML syntax +find_program(YAMLLINT_PATH yamllint) +if (YAMLLINT_PATH) + add_test( + NAME test-local-yaml-syntax + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-yaml-syntax" "${YAMLLINT_PATH}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) +else () + message(STATUS "Set YAMLLINT_PATH to the path of the yamllint executable to enable YAML syntax checks.") +endif () + +# add tidy check +add_test( + NAME test-local-tidy + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/tidy" --check + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" +) + +# add test for Perl syntax/style issues +find_program(PERLCRITIC_PATH perlcritic) +if (PERLCRITIC_PATH) + add_test( + NAME test-local-perl-style + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/check-perl-style" "${PERLCRITIC_PATH}" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) +else () + message(STATUS "Set PERLCRITIC_PATH to the path of the perlcritic executable to enable Perl syntax/style checks.") +endif () + +# add spell checking for test API documentation +find_program(PODSPELL_PATH podspell) +find_program(SPELL_PATH spell) +if (PODSPELL_PATH AND SPELL_PATH) + add_test( + NAME test-doc-testapi-spellchecking + COMMAND sh -c "\"${PODSPELL_PATH}\" \"${CMAKE_CURRENT_SOURCE_DIR}/testapi.pm\" | \"${SPELL_PATH}\"" + ) +else () + message(STATUS "Set PODSPELL_PATH/SPELL_PATH to the path of the podspell/spell executable to enable spell checking.") +endif () + +# add targets for invoking Perl test suite +find_program(PROVE_PATH prove) +if (PROVE_PATH) + set(INVOKE_TEST_ARGS --prove-tool "${PROVE_PATH}" --make-tool "${CMAKE_MAKE_PROGRAM}" --build-directory "${CMAKE_CURRENT_BINARY_DIR}") + add_test( + NAME test-perl-testsuite + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/invoke-tests" ${INVOKE_TEST_ARGS} + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) +else () + message(STATUS "Set PROVE_PATH to the path of the prove executable to enable running the Perl testsuite.") +endif () + +# add build system targets for invoking specific tests +add_custom_target(test-local COMMAND ${CMAKE_CTEST_COMMAND} -R "test-local-.*" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") +add_custom_target(test-doc COMMAND ${CMAKE_CTEST_COMMAND} -R "test-doc-.*" WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}") +add_custom_target(test-installed-files COMMAND ${CMAKE_CTEST_COMMAND} -R "test-installed-files" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_custom_target(test-perl-testsuite COMMAND ${CMAKE_CTEST_COMMAND} -R "test-perl-testsuite" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) +add_dependencies(test-perl-testsuite symlinks) + +# add target for computing test coverage of Perl test suite +find_program(COVER_PATH cover) +if (COVER_PATH AND PROVE_PATH) + add_custom_command( + COMMENT "Run Perl testsuite with coverage instrumentation" + COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/tools/invoke-tests" --coverage ${INVOKE_TEST_ARGS} + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/cover_db/structure" + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + ) + add_custom_command( + COMMENT "Generate coverage report (HTML)" + COMMAND "${COVER_PATH}" -report html_basic "${CMAKE_CURRENT_BINARY_DIR}/cover_db" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/cover_db/structure" + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/coverage.html" + ) + add_custom_target( + coverage-reset + COMMENT "Resetting previously gathered Perl test suite coverage" + COMMAND rm -r "${CMAKE_CURRENT_BINARY_DIR}/cover_db" + ) + add_custom_target( + coverage + COMMENT "Perl test suite coverage (HTML)" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/coverage.html" + ) + add_dependencies(coverage symlinks) + add_custom_target( + coverage-codecov + COMMENT "Perl test suite coverage (codecov)" + COMMAND "${COVER_PATH}" -report codecov "${CMAKE_CURRENT_BINARY_DIR}/cover_db" + DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/cover_db/structure" + ) + add_dependencies(coverage-codecov symlinks) +else () + message(STATUS "Set COVER_PATH to the path of the cover executable to enable coverage computition of the Perl testsuite.") +endif () diff --git a/t/27-make-update-deps.t b/t/27-make-update-deps.t index a652d9cf2e0..682f0c78e5f 100644 --- a/t/27-make-update-deps.t +++ b/t/27-make-update-deps.t @@ -26,16 +26,17 @@ if (not -e "$Bin/../.git") { exit; } -chdir "$Bin/.."; -my $make = "make update-deps"; -my @out = qx{$make}; -my $rc = $?; -die "Could not run $make: rc=$rc" if $rc; +my $build_dir = $ENV{OS_AUTOINST_BUILD_DIRECTORY} || "$Bin/.."; +my $make_tool = $ENV{OS_AUTOINST_MAKE_TOOL} || 'make'; +my $make_cmd = "$make_tool update-deps"; -my @status = grep { not m/^\?/ } qx{git status --porcelain}; +chdir $build_dir; +my @out = qx{$make_cmd}; +my $rc = $?; +die "Could not run $make_cmd: rc=$rc" if $rc; -ok(!@status, "No changed files after '$make'") - or diag @status; +my @status = grep { not m/^\?/ } qx{git -C "$Bin/.." status --porcelain}; +ok(!@status, "No changed files after '$make_cmd'") or diag @status; done_testing; diff --git a/tools/check-installed-files b/tools/check-installed-files index 8087849579c..c4f2c7d64c9 100755 --- a/tools/check-installed-files +++ b/tools/check-installed-files @@ -1,8 +1,9 @@ #!/bin/sh -e destdir="${destdir:-$(mktemp -d)}" -make install DESTDIR="$destdir" +checkoutdir="$(dirname "$0")/.." +env DESTDIR="$destdir" "${1:-make}" install installed_files=$(find "$destdir" -name '*.pm') -all_source_files=$(git ls-files "*.pm" 2>/dev/null || find "$(dirname "$0")/.." -name "*.pm" -printf "%P\n") +all_source_files=$(git -C "$checkoutdir" ls-files "*.pm" 2>/dev/null || find "$checkoutdir" -name "*.pm" -printf "%P\n") source_files=$(echo "$all_source_files" | grep -v -E '(ppmclibs/|t/|tools/)') for i in $source_files; do if ! echo "$installed_files" | grep -q "$i"; then diff --git a/tools/check-perl-style b/tools/check-perl-style new file mode 100755 index 00000000000..ba14d5ca4ce --- /dev/null +++ b/tools/check-perl-style @@ -0,0 +1,3 @@ +#!/bin/sh -e +export PERL5LIB=tools/lib/perlcritic:$PERL5LIB +"${1:-perlcritic}" --gentle --include Perl::Critic::Policy::HashKeyQuote "${2:-$PWD}" diff --git a/tools/check-yaml-syntax b/tools/check-yaml-syntax new file mode 100755 index 00000000000..8c9a33d965e --- /dev/null +++ b/tools/check-yaml-syntax @@ -0,0 +1,3 @@ +#!/bin/sh -e +# fall back to find if there is no git, e.g. in package builds +"${1:-yamllint}" --strict $(git ls-files "*.yml" "*.yaml" 2>/dev/null || find -name '*.y*ml') diff --git a/tools/invoke-tests b/tools/invoke-tests new file mode 100755 index 00000000000..9a114440449 --- /dev/null +++ b/tools/invoke-tests @@ -0,0 +1,47 @@ +#!/bin/bash -e + +source_directory=$(dirname "$0")/.. +source_directory=$(realpath "$source_directory") + +usage() { + echo "usage: $0 [options] [test-file...] +options: + --coverage record test coverage + --prove-tool prove tool to use (defaults to prove) + --make-tool make tool to use (defaults to make) + --build-directory build directory (defaults to $source_directory)" + exit +} + +# parse arguments +prove_path=prove make_path=make build_directory=$source_directory +opts=$(getopt --options h --longoptions help,coverage,prove-tool:,make-tool:,build-directory: --name "$(basename "$0")" -- "$@") || usage +eval set -- "$opts" +while [[ $# -gt 0 ]]; do + case "$1" in + -h | --help ) usage; shift ;; + --coverage ) WITH_COVER_OPTIONS=1; shift ;; + --prove-tool ) prove_path=$2; shift 2 ;; + --make-tool ) make_path=$2; shift 2 ;; + --build-directory ) build_directory=$2; shift 2 ;; + -- ) shift; break ;; + * ) break ;; + esac +done +[[ "$@" ]] && TESTS="$@" + +# set Perl module include path and coverage options +export PERL5LIB="$source_directory:$source_directory/ppmclibs:$source_directory/ppmclibs/blib/arch/auto/tinycv:$PERL5LIB" +if [[ $WITH_COVER_OPTIONS ]]; then + db="$build_directory/cover_db" + path_regex="^|$source_directory/|\\.\\./" + select="($path_regex)(OpenQA|backend|consoles|ppmclibs)/|($path_regex)isotovideo|($path_regex)[^/]+\.pm,-ignore,\.t|data/tests/|fake/tests/|$prove_path" + export PERL5OPT="-MDevel::Cover=-db,$db,-select,$select,-coverage,statement" +fi + +# set variables for tests which need to invoke the make tool +export OS_AUTOINST_BUILD_DIRECTORY=$build_directory +export OS_AUTOINST_MAKE_TOOL=$make_path + +# invoke tests within the t directory +cd t && "$prove_path" $PROVE_ARGS ${TESTS:-'-r'} diff --git a/tools/update-deps b/tools/update-deps index fa80d78bc2b..3d1edd70149 100755 --- a/tools/update-deps +++ b/tools/update-deps @@ -8,7 +8,6 @@ use Data::Dumper; use Mojo::File qw(path); use Getopt::Long; use FindBin qw($Bin); -use File::Spec; GetOptions( "help|h" => \my $help, @@ -19,7 +18,7 @@ GetOptions( usage(0) if $help; usage(1) unless $specfile; -my $scriptname = File::Spec->canonpath(__FILE__); +my $scriptname = path(__FILE__)->to_rel("$Bin/.."); my $yamlfile = "dependencies.yaml"; my $file = "$Bin/../$yamlfile"; my $cpanfile = "$Bin/../cpanfile";