Disable tests by default if tools are not found

If tools like `pg_regress` or `pg_isolation_regress` are not found, an
error is generated telling you that you need to disable the tests
explicitly using `REGRESS_CHECKS`. This is an inconvenience for the
user since in most cases you just want to build the system with what
you have.

This commit changes that by automatically disabling regression tests,
isolation tests, TAP tests, and formatting targets when tools cannot be
found, which means that users do not normally have to disable tests
explicitly. A notice is still printed that the targets are not added.

In addition, the commit adds an option `REQUIRE_ALL_TESTS` for the rare
case when you want to make sure that all tests are indeed executed (for
example, in CI runs), which by default is off.

.github/workflows/coverity.yaml

@@ -55,7 +55,7 @@ jobs:
     - name: Build TimescaleDB
       run: |
         PATH="$GITHUB_WORKSPACE/coverity/bin:$PATH"
-        ./bootstrap -DCMAKE_BUILD_TYPE=Release -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DTAP_CHECKS=off
+        ./bootstrap -DCMAKE_BUILD_TYPE=Release -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR
         cov-build --dir cov-int make -C build
 
     - name: Upload report

.github/workflows/linux-32bit-build-and-test.yaml

@@ -55,7 +55,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" -DPG_SOURCE_DIR=~/postgresql
+        ./bootstrap -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" -DPG_SOURCE_DIR=~/postgresql -DREQUIRE_ALL_TESTS=ON
         make -C build install
         chown -R postgres:postgres .
 

.github/workflows/linux-build-and-test.yaml

@@ -98,7 +98,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR ${{ matrix.tsdb_build_args }}
+        ./bootstrap -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR ${{ matrix.tsdb_build_args }} -DREQUIRE_ALL_TESTS=ON
         make -j $MAKE_JOBS -C build
         make -C build install
 

.github/workflows/sanitizer-build-and-test.yaml

@@ -82,7 +82,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF
+        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON
         make -j $MAKE_JOBS -C build
         make -C build install
 

.github/workflows/windows-build-and-test.yaml

@@ -47,7 +47,7 @@ jobs:
         choco feature disable --name=showDownloadProgress
         choco install postgresql${{ matrix.pg }} --version ${{ matrix.pkg_version }} --force -y --install-args="'--prefix $HOME\PostgreSQL\${{ matrix.pg }} --extract-only yes'"
     - name: Configure [${{ matrix.build_type }}]
-      run: cmake -B ${{ matrix.build_type }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DREGRESS_CHECKS=OFF -DPG_PATH="$HOME/PostgreSQL/${{ matrix.pg }}" -DOPENSSL_ROOT_DIR="$HOME/PostgreSQL/${{ matrix.pg }}" -DTAP_CHECKS=OFF
+      run: cmake -B ${{ matrix.build_type }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPG_PATH="$HOME/PostgreSQL/${{ matrix.pg }}" -DOPENSSL_ROOT_DIR="$HOME/PostgreSQL/${{ matrix.pg }}"
       # Build step: could potentially speed things up with --parallel
       # <num_cpu> or "-- -m:<num_cpu>", but MSBuild doesn't seem to be
       # able to realize interdependencies between targets, which leads

CMakeLists.txt

@@ -6,8 +6,26 @@ include(CheckCCompilerFlag)
 include(GitCommands)
 include(GenerateScripts)
 
+# This requires all tests to run. This defaults to OFF but can be enabled to
+# ensure that no tests are skipped because of missing tools.
+option(REQUIRE_ALL_TESTS "Require all tests to run." OFF)
+option(USE_OPENSSL "Enable use of OpenSSL if available" ON)
+option(SEND_TELEMETRY_DEFAULT "The default value for whether to send telemetry"
+       ON)
+option(REGRESS_CHECKS "PostgreSQL regress checks through installcheck" ON)
+option(
+  ENABLE_OPTIMIZER_DEBUG
+  "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG."
+  OFF)
+
+# Option to enable assertions. Note that if we include headers from a PostgreSQL
+# build that has assertions enabled, we might inherit that setting without
+# explicitly enabling assertions via the ASSERTIONS option defined here. Thus,
+# this option is mostly useful to enable assertions when the PostgreSQL we
+# compile against has it disabled.
+option(ASSERTIONS "Compile with assertion checks (default OFF)" OFF)
+
 # Function to call pg_config and extract values.
-#
 function(GET_PG_CONFIG var)
   set(_temp)
 
@@ -420,10 +438,7 @@ endif()
 if(NOT CLANG_FORMAT)
   find_program(DOCKER docker DOC "The path to docker")
 
-  if(NOT DOCKER)
-    message(
-      WARNING "clang-format is disabled (can't find clang-format or docker)")
-  else()
+  if(DOCKER)
     message(STATUS "Using docker based clang-format")
     add_custom_target(
       clang-format
@@ -462,6 +477,29 @@ if(TARGET clang-format OR TARGET cmake-format)
   endif()
 endif()
 
+if(REGRESS_CHECKS)
+  find_program(PG_REGRESS pg_regress
+               HINTS "${PG_BINDIR}" "${PG_PKGLIBDIR}/pgxs/src/test/regress/")
+
+  if(NOT PG_REGRESS)
+    message(STATUS "Regress checks disabled: program 'pg_regress' not found")
+  endif()
+
+  find_program(
+    PG_ISOLATION_REGRESS
+    NAMES pg_isolation_regress
+    HINTS ${PG_BINDIR} ${PG_PKGLIBDIR}/pgxs/src/test/isolation
+          ${PG_SOURCE_DIR}/src/test/isolation ${BINDIR})
+
+  if(NOT PG_ISOLATION_REGRESS)
+    message(
+      STATUS
+        "Isolation regress checks disabled: 'pg_isolation_regress' not found")
+  endif()
+else()
+  message(STATUS "Regress checks and isolation checks disabled")
+endif()
+
 if(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang")
   set(LINTER_DEFAULT ON)
 else()
@@ -487,22 +525,6 @@ if(LINTER)
   endif(CLANG_TIDY)
 endif(LINTER)
 
-option(USE_OPENSSL "Enable use of OpenSSL if available" ON)
-option(SEND_TELEMETRY_DEFAULT "The default value for whether to send telemetry"
-       ON)
-option(REGRESS_CHECKS "PostgreSQL regress checks through installcheck" ON)
-option(
-  ENABLE_OPTIMIZER_DEBUG
-  "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG."
-  OFF)
-
-# Option to enable assertions. Note that if we include headers from a PostgreSQL
-# build that has assertions enabled, we might inherit that setting without
-# explicitly enabling assertions via the ASSERTIONS option defined here. Thus,
-# this option is mostly useful to enable assertions when the PostgreSQL we
-# compile against has it disabled.
-option(ASSERTIONS "Compile with assertion checks (default OFF)" OFF)
-
 if(NOT EXISTS ${PG_INCLUDEDIR}/pg_config.h)
   message(
     FATAL_ERROR
@@ -517,8 +539,9 @@ string(REGEX MATCH "#define USE_ASSERT_CHECKING 1" PG_USE_ASSERT_CHECKING
 
 if(PG_USE_ASSERT_CHECKING AND NOT ASSERTIONS)
   message(
-    "Assertion checks are OFF although enabled in PostgreSQL build (pg_config.h). "
-    "The PostgreSQL setting for assertions will take precedence.")
+    STATUS
+      "Assertion checks are OFF although enabled in PostgreSQL build (pg_config.h). "
+      "The PostgreSQL setting for assertions will take precedence.")
 elseif(ASSERTIONS)
   message(STATUS "Assertion checks are ON")
   add_compile_definitions(USE_ASSERT_CHECKING=1)
@@ -601,10 +624,8 @@ if(TAP_CHECKS)
       PATHS "/usr/bin")
 
     if(NOT PROVE)
-      message(
-        FATAL_ERROR
-          "The prove binary was not found. Disable TAP tests with -DTAP_CHECKS=OFF."
-      )
+      message(STATUS "Not running TAP tests: 'prove' binary not found.")
+      set(TAP_CHECKS OFF)
     endif()
 
     # Check for the IPC::Run module
@@ -614,18 +635,14 @@ if(TAP_CHECKS)
       RESULT_VARIABLE PERL_MODULE_STATUS)
 
     if(PERL_MODULE_STATUS)
-      message(
-        FATAL_ERROR
-          "IPC::Run Perl module not found. Disable TAP tests with -DTAP_CHECKS=OFF."
-      )
+      message(STATUS "Not running TAP tests: IPC::Run Perl module not found.")
+      set(TAP_CHECKS OFF)
     endif()
   else()
-    message(
-      FATAL_ERROR
-        "Perl not found. Disable TAP tests with -DTAP_CHECKS=OFF or set -DPERL_EXECUTABLE=<path/to/perl>. Requires Perl 5.8 or greater (http://perlbrew.pl)."
-    )
+    message(STATUS "Not running TAP tests: Perl not found.")
+    set(TAP_CHECKS OFF)
   endif()
-endif(TAP_CHECKS)
+endif()
 
 if(UNIX)
   add_subdirectory(scripts)

appveyor.yml

@@ -131,7 +131,7 @@ build_script:
 
     # build timescale
 
-     .\bootstrap -DUSE_OPENSSL=0 -DPG_PATH="C:\Program Files\PostgreSQL\12" -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CONFIGURATION_TYPES=Debug -DCMAKE_C_FLAGS=/MP -DTAP_CHECKS=OFF
+     .\bootstrap -DUSE_OPENSSL=0 -DPG_PATH="C:\Program Files\PostgreSQL\12" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CONFIGURATION_TYPES=Debug -DCMAKE_C_FLAGS=/MP
 
     # Filter ssl and local configuration from pg_hba.conf file since
     # we have turned off SSL and local (unix domain socket)
@@ -180,7 +180,7 @@ build_script:
 
     docker exec -it pgregress /bin/bash -c "git clone  --depth 1 https://github.com/timescale/timescaledb.git /timescaledb && cd /timescaledb && git fetch -q origin $commit && git checkout -qf FETCH_HEAD"
 
-    docker exec -it pgregress /bin/bash -c "cd /timescaledb && ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DREGRESS_CHECKS=OFF -DPG_REGRESS=/usr/local/bin/pg_regress -DTEST_PGHOST=docker.for.win.localhost -DTEST_PGPORT_LOCAL=55432 -DTAP_CHECKS=OFF"
+    docker exec -it pgregress /bin/bash -c "cd /timescaledb && ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DTEST_PGHOST=docker.for.win.localhost -DTEST_PGPORT_LOCAL=55432"
 
 test_script:
 - ps: >-

scripts/docker-build.sh

@@ -78,7 +78,7 @@ build_timescaledb()
 
     # Build and install the extension with debug symbols and assertions
     tar -c -C ${BASE_DIR} {cmake,src,sql,test,scripts,tsl,version.config,CMakeLists.txt,timescaledb.control.in} | docker cp - ${BUILD_CONTAINER_NAME}:/build/
-    docker exec -u root ${BUILD_CONTAINER_NAME} /bin/bash -c "cd /build/debug && cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DUSE_OPENSSL=${USE_OPENSSL} -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DTAP_CHECKS=OFF /src && make && make install && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample && echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample && cd / && rm -rf /build"
+    docker exec -u root ${BUILD_CONTAINER_NAME} /bin/bash -c "cd /build/debug && cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DUSE_OPENSSL=${USE_OPENSSL} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} /src && make && make install && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample && echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample && cd / && rm -rf /build"
     if [ $? -ne 0 ]; then
       echo "Building timescaledb failed"
       return 1

scripts/docker-run-abi-test.sh

@@ -38,7 +38,7 @@ create_base_container() {
   docker exec -u root $1 /bin/bash -c "apk add --no-cache --virtual .build-deps coreutils dpkg-dev gcc libc-dev make util-linux-dev diffutils cmake bison flex openssl-dev perl perl-utils perl-test-harness-utils perl-ipc-run && mkdir -p /build/debug "
 
   # Copy the source files to build directory
-  docker exec -u root $1 /bin/bash -c "cp -a /src/{src,sql,scripts,test,tsl,cmake,CMakeLists.txt,timescaledb.control.in,version.config} /build/ && cd /build/debug/ && CFLAGS=-Werror cmake .. -DCMAKE_BUILD_TYPE=Debug -DREGRESS_CHECKS=OFF && make -C /build/debug install && chown -R postgres /build"
+  docker exec -u root $1 /bin/bash -c "cp -a /src/{src,sql,scripts,test,tsl,cmake,CMakeLists.txt,timescaledb.control.in,version.config} /build/ && cd /build/debug/ && CFLAGS=-Werror cmake .. -DCMAKE_BUILD_TYPE=Debug && make -C /build/debug install && chown -R postgres /build"
 }
 
 create_base_container $CONTAINER_NAME_COMPILE $PG_IMAGE_TAG_COMPILE

scripts/test_sanitizers.sh

@@ -79,7 +79,7 @@ docker exec -i -u postgres -w /tsdb_build/timescaledb timescaledb-san /bin/bash
 export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -O2"
 export PG_SOURCE_DIR="/usr/src/postgresql/"
 export BUILD_FORCE_REMOVE=true
-./bootstrap -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE='Debug' -DTEST_GROUP_SIZE=1
+./bootstrap -DCMAKE_BUILD_TYPE='Debug' -DTEST_GROUP_SIZE=1
 cd build
 make
 EOF

test/CMakeLists.txt

@@ -2,21 +2,16 @@ set(PRIMARY_TEST_DIR ${CMAKE_CURRENT_LIST_DIR})
 set(PRIMARY_TEST_DIR
     ${CMAKE_CURRENT_LIST_DIR}
     PARENT_SCOPE)
+
 set(_local_install_checks)
 set(_install_checks)
 
 # Testing support
-find_program(PG_REGRESS pg_regress
-             HINTS "${PG_BINDIR}" "${PG_PKGLIBDIR}/pgxs/src/test/regress/")
-
-find_program(
-  PG_ISOLATION_REGRESS
-  NAMES pg_isolation_regress
-  HINTS ${PG_BINDIR} ${PG_PKGLIBDIR}/pgxs/src/test/isolation
-        ${PG_SOURCE_DIR}/src/test/isolation ${BINDIR})
-
 include(test-defs.cmake)
 
+# No checks for REGRESS_CHECKS needed here since all the checks are done in the
+# parent CMakeLists.txt.
+
 if(PG_REGRESS)
   message(STATUS "Using pg_regress ${PG_REGRESS}")
 
@@ -41,16 +36,9 @@ if(PG_REGRESS)
 
   list(APPEND _local_install_checks regresschecklocal)
   list(APPEND _install_checks regresscheck)
-elseif(REGRESS_CHECKS)
+elseif(REQUIRE_ALL_TESTS)
   message(
-    FATAL_ERROR
-      "Program 'pg_regress' not found, but regressions checks were required.\n"
-      "Skip regression checks using -DREGRESS_CHECKS=OFF")
-else()
-  message(
-    STATUS
-      "Skipping isolation regress checks since program 'pg_regress' was not found"
-  )
+    FATAL_ERROR "All tests were required but 'pg_regress' could not be found")
 endif()
 
 if(PG_ISOLATION_REGRESS)
@@ -79,21 +67,10 @@ if(PG_ISOLATION_REGRESS)
 
   list(APPEND _local_install_checks isolationchecklocal)
   list(APPEND _install_checks isolationcheck)
-elseif(REGRESS_CHECKS)
+elseif(REQUIRE_ALL_TESTS)
   message(
     FATAL_ERROR
-      "Program 'pg_isolation_regress' not found, but regressions checks were required.\n"
-      "Skip regression checks using -DREGRESS_CHECKS=OFF")
-else()
-  message(
-    STATUS
-      "Skipping isolation regress checks since program 'pg_isolation_regress' was not found"
-  )
-endif()
-
-# installcheck starts up new temporary instances for testing code
-if(NOT APACHE_ONLY)
-  list(APPEND _install_checks installcheck-t)
+      "All tests were required but 'pg_isolation_regress' could not be found")
 endif()
 
 if(_install_checks)

tsl/test/CMakeLists.txt

@@ -5,6 +5,8 @@ add_subdirectory(ssl)
 set(_local_install_checks)
 set(_install_checks)
 
+# No checks for REGRESS_CHECKS needed here since all the checks are done in the
+# parent CMakeLists.txt.
 if(PG_REGRESS)
   add_custom_target(
     regresscheck-t
@@ -51,22 +53,29 @@ if(PG_REGRESS)
       ${PG_REGRESS_OPTS_LOCAL_INSTANCE}
     USES_TERMINAL)
 
-  if(TAP_CHECKS)
-    add_custom_target(
-      provecheck
-      COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/tmp_check
-      COMMAND
-        CONFDIR=${CMAKE_BINARY_DIR}/tsl/test PATH="${PG_BINDIR}:$ENV{PATH}"
-        PG_REGRESS=${PG_REGRESS} SRC_DIR=${PG_SOURCE_DIR}
-        CM_SRC_DIR=${CMAKE_SOURCE_DIR} PG_LIBDIR=${PG_LIBDIR}
-        ${PRIMARY_TEST_DIR}/pg_prove.sh
-      USES_TERMINAL)
-    list(APPEND _install_checks provecheck)
-  endif()
-
   list(APPEND _install_checks regresscheck-shared)
   list(APPEND _local_install_checks regresschecklocal-shared)
+elseif(REQUIRE_ALL_TESTS)
+  message(
+    FATAL_ERROR "All tests were required but 'pg_regress' could not be found")
+endif()
 
+if(TAP_CHECKS)
+  add_custom_target(
+    provecheck
+    COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/tmp_check
+    COMMAND
+      CONFDIR=${CMAKE_BINARY_DIR}/tsl/test PATH="${PG_BINDIR}:$ENV{PATH}"
+      PG_REGRESS=${PG_REGRESS} SRC_DIR=${PG_SOURCE_DIR}
+      CM_SRC_DIR=${CMAKE_SOURCE_DIR} PG_LIBDIR=${PG_LIBDIR}
+      ${PRIMARY_TEST_DIR}/pg_prove.sh
+    USES_TERMINAL)
+  list(APPEND _install_checks provecheck)
+elseif(REQUIRE_ALL_TESTS)
+  message(
+    FATAL_ERROR
+      "All tests were required but TAP_CHECKS was off (see previous messages why)"
+  )
 endif()
 
 if(PG_ISOLATION_REGRESS)
@@ -94,6 +103,10 @@ if(PG_ISOLATION_REGRESS)
 
   list(APPEND _local_install_checks isolationchecklocal-t)
   list(APPEND _install_checks isolationcheck-t)
+elseif(REQUIRE_ALL_TESTS)
+  message(
+    FATAL_ERROR
+      "All tests were required but 'pg_isolation_regress' could not be found")
 endif()
 
 add_subdirectory(shared)
@@ -102,8 +115,15 @@ add_subdirectory(isolation)
 add_subdirectory(t)
 
 # installchecklocal tests against an existing postgres instance
-add_custom_target(installchecklocal-t DEPENDS ${_local_install_checks})
-add_custom_target(installcheck-t DEPENDS ${_install_checks})
+if(_local_install_checks)
+  add_custom_target(installchecklocal-t DEPENDS ${_local_install_checks})
+  add_dependencies(installchecklocal installchecklocal-t)
+endif()
+
+if(_install_checks)
+  add_custom_target(installcheck-t DEPENDS ${_install_checks})
+  add_dependencies(installcheck installcheck-t)
+endif()
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
   add_subdirectory(src)