add ctest as test driver

.github/workflows/tests.yaml

@@ -99,10 +99,10 @@ jobs:
           GITHUB_USER: ${{ github.actor }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CONTAINER: ${{ matrix.dockerfile }}
-        run: docker-compose build test-$CONTAINER
+        run: docker-compose build test-${{ env.CONTAINER }}
 
       - name: Test ${{ matrix.name }}
-        run: docker-compose run test-$CONTAINER
+        run: docker-compose run test-${{ env.CONTAINER }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           CONTAINER: ${{ matrix.dockerfile }}

CMakeLists.txt

@@ -1,4 +1,5 @@
 cmake_minimum_required (VERSION 3.22)
+enable_testing()
 
 set (CMAKE_EXPORT_COMPILE_COMMANDS ON)
 

doc/devel/contrib/development.md

@@ -2,7 +2,7 @@
 
 ## Satisfy the Requirements:
 
- * CMake 3.0 or later
+ * CMake 3.22 or later
  * gcc 7.0 or later, clang 6.0 or later, or a compiler with full C++17 support
  * libuuid (if not on macOS)
  * Rust 1.64.0 or higher (hint: use https://rustup.rs/ instead of using your system's package manager)
@@ -49,28 +49,47 @@ cmake --build build-clang
 ```
 
 ## Run the Test Suite:
-First switch to the test directory:
-
-```
-    $ cd build/test
-```
-Then you can run all tests, showing details, with
-```
-    $ make VERBOSE=1 test
-```
-Alternately, run the tests with the details hidden in `all.log`:
+For running the test suite [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) is used.
+Before one can run the test suite the `task_executable` must be built.
+After that also the `build_tests` executable must be build, which can be done over:
+```sh
+cmake --build build --target build_tests
 ```
-    $ ./run_all
+Again you may also use the `-j <number-of-jobs>` option for parallel builds.
+
+Now one can invoke `ctest` to run the tests.
+```sh
+ctest --test-dir build
 ```
-Either way, you can get a summary of any test failures with:
+This would run all the test in serial and might take some time.
+
+### Running tests in parallel
+```sh
+ctest --test-dir build -j <number-of-jobs>
 ```
-    $ ./problems
+
+Further it is adviced to add the `--output-on-failure` option to `ctest`, to recieve a verbose output if a test is failing as well as the `--rerun-failed` flag, to invoke in subsequent runs only the failed ones.
+
+### Running specific tests
+For this case one can use the `-R <regex>` or `--tests-regex <regex>`  option to run only the tests matching the regular expression.
+Running only the `cpp` tests can then be achieved over
+```sh
+ctest --test-dir build -R cpp
 ```
-You can run a single test suite, with source file `foo.test.cpp` or `foo.test.py`, with
+or running the `variant_*` tests
+```sh
+ctest --test-dir build -R variant
 ```
-    $ make foo.test
+
+### Repeating a test case
+In order to find sporadic test failures the `--repeat` flag can be used.
+```sh
+ctest --test-dir build -R cpp --repeat-until-fail 10
 ```
 
+There are more options to `ctest` such as `--progress`, allowing to have a less verbose output.
+They can be found in the [ctest](https://cmake.org/cmake/help/latest/manual/ctest.1.html) man page.
+
 Note that any development should be performed using a git clone, and the current development branch.
 The source tarballs do not reflect HEAD, and do not contain the test suite.
 Follow the [GitHub flow](https://docs.github.com/en/get-started/quickstart/github-flow) for creating a pull request.

test/CMakeLists.txt

@@ -1,13 +1,5 @@
 cmake_minimum_required (VERSION 3.22)
 
-# This is a work-around for the following CMake issue:
-# https://gitlab.kitware.com/cmake/cmake/issues/16062
-# The issue has been fixed in CMake 3.11.0; the policy is set
-# to OLD for compatibility with older versions of CMake only.
-if(POLICY CMP0037 AND ${CMAKE_VERSION} VERSION_LESS "3.11.0")
-  cmake_policy(SET CMP0037 OLD)
-endif()
-
 include_directories (${CMAKE_SOURCE_DIR}
                      ${CMAKE_SOURCE_DIR}/src
                      ${CMAKE_SOURCE_DIR}/src/tc/lib
@@ -51,30 +43,23 @@ set (test_SRCS
   view.test.cpp
   )
 
-add_custom_target (test ./run_all --verbose
-                        DEPENDS ${test_SRCS} task_executable
-                        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
-
 add_custom_target (build_tests DEPENDS ${test_SRCS}
                                WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/test)
 
 foreach (src_FILE ${test_SRCS})
   add_executable (${src_FILE} ${src_FILE} test.cpp)
   target_link_libraries (${src_FILE} task tc commands columns libshared task tc commands columns libshared task commands columns libshared ${TASK_LIBRARIES})
+  add_dependencies (${src_FILE} task_executable)
   if (DARWIN)
     target_link_libraries (${src_FILE} "-framework CoreFoundation -framework Security -framework SystemConfiguration")
   endif (DARWIN)
 
-  # Add a custom `foo.test` target.
-  string(REGEX REPLACE "\\.[^.]*$" "" test_target ${src_FILE})
-  add_custom_target (${test_target}
-                     ./${src_FILE}
-                     DEPENDS ${src_FILE}
-                     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
+  add_test(NAME ${src_FILE}
+    COMMAND ${src_FILE}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
 endforeach (src_FILE)
 
-configure_file(run_all run_all COPYONLY)
-configure_file(problems problems COPYONLY)
 configure_file(bash_tap.sh bash_tap.sh COPYONLY)
 configure_file(bash_tap_tw.sh bash_tap_tw.sh COPYONLY)
 
@@ -205,15 +190,8 @@ set (pythonTests
 foreach (python_Test ${pythonTests})
   configure_file(${python_Test} ${python_Test} COPYONLY)
 
-  # Add a custom `foo.test` target.
-  string(REGEX REPLACE "\\.[^.]*$" "" test_target ${python_Test})
-  add_custom_target (${test_target}
-                     ./${python_Test}
-                     DEPENDS ${python_Test}
-                     WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test)
+  add_test(NAME ${python_Test}
+    COMMAND ${Python_EXECUTABLE} ${python_Test}
+    WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
 endforeach(python_Test)
-
-#SET(CMAKE_BUILD_TYPE gcov)
-#SET(CMAKE_CXX_FLAGS_GCOV "--coverage")
-#SET(CMAKE_C_FLAGS_GCOV "--coverage")
-#SET(CMAKE_EXE_LINKER_FLAGS_GCOV "--coverage")

test/README → test/README.md

@@ -1,36 +1,18 @@
-README
-======
-
-This is the task.git/test/README file, and contains notes about the Taskwarrior
-test suite.
-
-
-Running Tests
--------------
-
+## Running Tests
 Do this to run all tests:
+```shell
+cmake --build build --target build_tests
+ctest --test-dir build
+```
 
-  $ cd test && make && ./run_all && ./problems
-
-All unit tests produce TAP (Test Anything Protocol) output, and are run by the
-'run_all' test harness.
-
-The 'run_all' script produces an 'all.log' file which is the accumulated output
-of all tests. Before executing 'run_all' you need to compile the C++ unit
-tests, by running 'make' in the 'test' directory.
+All tests produce TAP (Test Anything Protocol) output.
+In order to run the tests in parallel add the `--parallel <# of threads>` or shortly `-j <# of threads>` option to `ctest`.
+Depending on your IDE, all tests might also be available under the `All CTest` target.
+Keep in mind that the tests are not automatically rebuild if a source file is changes, it requires a manual rebuild.
 
-The script 'problems' will list all the tests that fail, with a count of the
-failing tests, once you have run all the tests and produced an 'all.log' file.
+Please also have a look at [development.md](../doc/devel/contrib/development.md) for more information on how to run tests as well as some information about `ctest`.
 
-Any TAP harness may be used.
-
-Note that adding the '--serial' option to ./run_all, all tests are executed
-serially.  The default runs Python, C++ and Bash tests in parallel. Using
-'--serial' will make for a slower test run.
-
-
-Architecture
-------------
+## Architecture
 
 There are three varieties of tests:
 
@@ -45,17 +27,18 @@ There are three varieties of tests:
     tests are small, quick tests, not intended to be permanent.
 
 All tests are named with the pattern '*.test.py', '*.test.sh', or '*.test.cpp',
-and any other forms are not run by the test harness. Additionally a test must
-be set executable (chmod +x) for it to be run. In the case of Python tests one
-can still run them manually by launching them with 'python testname.test.py' or
-simply './testname.test.py'. It also allows us to keep tests submitted for bugs
-that are not scheduled to be fixed in the upcoming release, and we don't want
-the failing tests to prevent us from seeing 100% pass rate for the bugs we
-*have* fixed.
+and any other forms are not run by the test harness.
+In the case of Python tests one can still run them manually by launching them with 'python testname.test.py' or simply './testname.test.py'.
 
+If a test is failing and can not be fixed, it can be marked as `WILL_FAIL` in the `CMakeLists.txt` file.
+See the [WILL_FAIL](https://cmake.org/cmake/help/latest/prop_test/WILL_FAIL.html) documentation for more information.
+However, please keep in mind that such tests should be fixed as soon as possible as well as proper documentation should be added to the issue tracker.
 
-Goals
------
+It also allows us to keep tests submitted for bugs that are not scheduled to be fixed in the upcoming release, and we don't want
+the failing tests to prevent us from seeing 100% pass rate for the bugs we *have* fixed.
+
+
+## Goals
 
 The test suite is evolving, and becoming a better tool for determining whether
 code is ready for release. There are goals that shape these changes, and they
@@ -71,16 +54,14 @@ are:
     There is simply no point in testing a feature twice, in the same manner.
 
 
-What Makes a Good Test
-----------------------
+## What Makes a Good Test
 
 A good test ensures that a feature is functioning as expected, and contains
 both positive and negative aspects, or in other words looks for expected
 behavior as well as looking for the absence of unexpected behavior.
 
 
-Conventions for writing a test
-------------------------------
+## Conventions for writing a test
 
 If you wish to contribute tests, please consider the following guidelines:
 
@@ -109,14 +90,12 @@ If you wish to contribute tests, please consider the following guidelines:
     a live test that is skipped, than no test.
 
 
-How to Submit a Test Change/Addition
-------------------------------------
+## How to Submit a Test Change/Addition
 
 Mail it to support@gothenburgbitfactory.org, or attach it to an open bug.
 
 
-Wisdom
-------
+## Wisdom
 
 Here are some guildelines that may help:
 
@@ -146,8 +125,7 @@ Here are some guildelines that may help:
     are reported.
 
 
-TODO
-----
+## TODO
 
 For anyone looking for test-related tasks to take on, here are some suggestions:
 
@@ -161,6 +139,4 @@ For anyone looking for test-related tasks to take on, here are some suggestions:
 
   * All the attribute modifiers need to be tested, only a few are.
 
-  * Aliases are not well tested, and fragile.
-
----
+  * Aliases are not well tested, and fragile.

test/docker/arch

@@ -26,7 +26,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test/
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/debianstable

@@ -27,7 +27,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/debiantesting

@@ -27,7 +27,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/fedora39

@@ -26,7 +26,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/fedora40

@@ -26,7 +26,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/opensuse

@@ -25,7 +25,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/ubuntu2004

@@ -34,7 +34,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed

test/docker/ubuntu2204

@@ -27,7 +27,5 @@ RUN cmake --install build
 RUN task --version
 
 # Setup tests
-WORKDIR /root/code/build/test
-RUN make -j8
-
-CMD ["bash", "-c", "./run_all -v ; cat all.log | grep 'not ok' ; ./problems"]
+RUN cmake --build build --target build_tests -j 8
+CMD ctest --test-dir build -j 8 --output-on-failure --rerun-failed