Disable tests by default if tools are not found

.github/workflows/coverity.yaml

@@ -55,7 +55,7 @@ jobs:
     - name: Build TimescaleDB
       run: |
         PATH="$GITHUB_WORKSPACE/coverity/bin:$PATH"
-        ./bootstrap -DCMAKE_BUILD_TYPE=Release -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DTAP_CHECKS=off
+        ./bootstrap -DCMAKE_BUILD_TYPE=Release -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR
         cov-build --dir cov-int make -C build
 
     - name: Upload report

.github/workflows/linux-32bit-build-and-test.yaml

@@ -55,7 +55,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" -DPG_SOURCE_DIR=~/postgresql
+        ./bootstrap -DCMAKE_BUILD_TYPE="${{ matrix.build_type }}" -DPG_SOURCE_DIR=~/postgresql -DREQUIRE_ALL_TESTS=ON
         make -C build install
         chown -R postgres:postgres .
 

.github/workflows/linux-build-and-test.yaml

@@ -98,7 +98,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR ${{ matrix.tsdb_build_args }}
+        ./bootstrap -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR ${{ matrix.tsdb_build_args }} -DREQUIRE_ALL_TESTS=ON
         make -j $MAKE_JOBS -C build
         make -C build install
 

.github/workflows/sanitizer-build-and-test.yaml

@@ -82,7 +82,7 @@ jobs:
 
     - name: Build TimescaleDB
       run: |
-        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF
+        ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DPG_SOURCE_DIR=~/$PG_SRC_DIR -DPG_PATH=~/$PG_INSTALL_DIR -DCODECOVERAGE=OFF -DREQUIRE_ALL_TESTS=ON
         make -j $MAKE_JOBS -C build
         make -C build install
 

.github/workflows/windows-build-and-test.yaml

@@ -47,7 +47,7 @@ jobs:
         choco feature disable --name=showDownloadProgress
         choco install postgresql${{ matrix.pg }} --version ${{ matrix.pkg_version }} --force -y --install-args="'--prefix $HOME\PostgreSQL\${{ matrix.pg }} --extract-only yes'"
     - name: Configure [${{ matrix.build_type }}]
-      run: cmake -B ${{ matrix.build_type }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DREGRESS_CHECKS=OFF -DPG_PATH="$HOME/PostgreSQL/${{ matrix.pg }}" -DOPENSSL_ROOT_DIR="$HOME/PostgreSQL/${{ matrix.pg }}" -DTAP_CHECKS=OFF
+      run: cmake -B ${{ matrix.build_type }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DPG_PATH="$HOME/PostgreSQL/${{ matrix.pg }}" -DOPENSSL_ROOT_DIR="$HOME/PostgreSQL/${{ matrix.pg }}"
       # Build step: could potentially speed things up with --parallel
       # <num_cpu> or "-- -m:<num_cpu>", but MSBuild doesn't seem to be
       # able to realize interdependencies between targets, which leads

CMakeLists.txt

@@ -6,8 +6,26 @@ include(CheckCCompilerFlag)
 include(GitCommands)
 include(GenerateScripts)
 
+# This requires all tests to run. This defaults to OFF but can be enabled to
+# ensure that no tests are skipped because of missing tools.
+option(REQUIRE_ALL_TESTS "Require all tests to run." OFF)
+option(USE_OPENSSL "Enable use of OpenSSL if available" ON)
+option(SEND_TELEMETRY_DEFAULT "The default value for whether to send telemetry"
+       ON)
+option(REGRESS_CHECKS "PostgreSQL regress checks through installcheck" ON)
+option(
+  ENABLE_OPTIMIZER_DEBUG
+  "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG."
+  OFF)
+
+# Option to enable assertions. Note that if we include headers from a PostgreSQL
+# build that has assertions enabled, we might inherit that setting without
+# explicitly enabling assertions via the ASSERTIONS option defined here. Thus,
+# this option is mostly useful to enable assertions when the PostgreSQL we
+# compile against has it disabled.
+option(ASSERTIONS "Compile with assertion checks (default OFF)" OFF)
+
 # Function to call pg_config and extract values.
-#
 function(GET_PG_CONFIG var)
   set(_temp)
 
@@ -420,10 +438,7 @@ endif()
 if(NOT CLANG_FORMAT)
   find_program(DOCKER docker DOC "The path to docker")
 
-  if(NOT DOCKER)
-    message(
-      WARNING "clang-format is disabled (can't find clang-format or docker)")
-  else()
+  if(DOCKER)
     message(STATUS "Using docker based clang-format")
     add_custom_target(
       clang-format
@@ -462,6 +477,29 @@ if(TARGET clang-format OR TARGET cmake-format)
   endif()
 endif()
 
+if(REGRESS_CHECKS)
+  find_program(PG_REGRESS pg_regress
+               HINTS "${PG_BINDIR}" "${PG_PKGLIBDIR}/pgxs/src/test/regress/")
+
+  if(NOT PG_REGRESS)
+    message(STATUS "Regress checks disabled: program 'pg_regress' not found")
+  endif()
+
+  find_program(
+    PG_ISOLATION_REGRESS
+    NAMES pg_isolation_regress
+    HINTS ${PG_BINDIR} ${PG_PKGLIBDIR}/pgxs/src/test/isolation
+          ${PG_SOURCE_DIR}/src/test/isolation ${BINDIR})
+
+  if(NOT PG_ISOLATION_REGRESS)
+    message(
+      STATUS
+        "Isolation regress checks disabled: 'pg_isolation_regress' not found")
+  endif()
+else()
+  message(STATUS "Regress checks and isolation checks disabled")
+endif()
+
 if(CMAKE_C_COMPILER_ID MATCHES "Clang|AppleClang")
   set(LINTER_DEFAULT ON)
 else()
@@ -487,22 +525,6 @@ if(LINTER)
   endif(CLANG_TIDY)
 endif(LINTER)
 
-option(USE_OPENSSL "Enable use of OpenSSL if available" ON)
-option(SEND_TELEMETRY_DEFAULT "The default value for whether to send telemetry"
-       ON)
-option(REGRESS_CHECKS "PostgreSQL regress checks through installcheck" ON)
-option(
-  ENABLE_OPTIMIZER_DEBUG
-  "Enable OPTIMIZER_DEBUG when building. Requires Postgres server to be built with OPTIMIZER_DEBUG."
-  OFF)
-
-# Option to enable assertions. Note that if we include headers from a PostgreSQL
-# build that has assertions enabled, we might inherit that setting without
-# explicitly enabling assertions via the ASSERTIONS option defined here. Thus,
-# this option is mostly useful to enable assertions when the PostgreSQL we
-# compile against has it disabled.
-option(ASSERTIONS "Compile with assertion checks (default OFF)" OFF)
-
 if(NOT EXISTS ${PG_INCLUDEDIR}/pg_config.h)
   message(
     FATAL_ERROR
@@ -517,8 +539,9 @@ string(REGEX MATCH "#define USE_ASSERT_CHECKING 1" PG_USE_ASSERT_CHECKING
 
 if(PG_USE_ASSERT_CHECKING AND NOT ASSERTIONS)
   message(
-    "Assertion checks are OFF although enabled in PostgreSQL build (pg_config.h). "
-    "The PostgreSQL setting for assertions will take precedence.")
+    STATUS
+      "Assertion checks are OFF although enabled in PostgreSQL build (pg_config.h). "
+      "The PostgreSQL setting for assertions will take precedence.")
 elseif(ASSERTIONS)
   message(STATUS "Assertion checks are ON")
   add_compile_definitions(USE_ASSERT_CHECKING=1)
@@ -601,10 +624,8 @@ if(TAP_CHECKS)
       PATHS "/usr/bin")
 
     if(NOT PROVE)
-      message(
-        FATAL_ERROR
-          "The prove binary was not found. Disable TAP tests with -DTAP_CHECKS=OFF."
-      )
+      message(STATUS "Not running TAP tests: 'prove' binary not found.")
+      set(TAP_CHECKS OFF)
     endif()
 
     # Check for the IPC::Run module
@@ -614,18 +635,14 @@ if(TAP_CHECKS)
       RESULT_VARIABLE PERL_MODULE_STATUS)
 
     if(PERL_MODULE_STATUS)
-      message(
-        FATAL_ERROR
-          "IPC::Run Perl module not found. Disable TAP tests with -DTAP_CHECKS=OFF."
-      )
+      message(STATUS "Not running TAP tests: IPC::Run Perl module not found.")
+      set(TAP_CHECKS OFF)
     endif()
   else()
-    message(
-      FATAL_ERROR
-        "Perl not found. Disable TAP tests with -DTAP_CHECKS=OFF or set -DPERL_EXECUTABLE=<path/to/perl>. Requires Perl 5.8 or greater (http://perlbrew.pl)."
-    )
+    message(STATUS "Not running TAP tests: Perl not found.")
+    set(TAP_CHECKS OFF)
   endif()
-endif(TAP_CHECKS)
+endif()
 
 if(UNIX)
   add_subdirectory(scripts)

appveyor.yml

@@ -131,7 +131,7 @@ build_script:
 
     # build timescale
 
-     .\bootstrap -DUSE_OPENSSL=0 -DPG_PATH="C:\Program Files\PostgreSQL\12" -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CONFIGURATION_TYPES=Debug -DCMAKE_C_FLAGS=/MP -DTAP_CHECKS=OFF
+     .\bootstrap -DUSE_OPENSSL=0 -DPG_PATH="C:\Program Files\PostgreSQL\12" -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CONFIGURATION_TYPES=Debug -DCMAKE_C_FLAGS=/MP
 
     # Filter ssl and local configuration from pg_hba.conf file since
     # we have turned off SSL and local (unix domain socket)
@@ -180,7 +180,7 @@ build_script:
 
     docker exec -it pgregress /bin/bash -c "git clone  --depth 1 https://github.com/timescale/timescaledb.git /timescaledb && cd /timescaledb && git fetch -q origin $commit && git checkout -qf FETCH_HEAD"
 
-    docker exec -it pgregress /bin/bash -c "cd /timescaledb && ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DREGRESS_CHECKS=OFF -DPG_REGRESS=/usr/local/bin/pg_regress -DTEST_PGHOST=docker.for.win.localhost -DTEST_PGPORT_LOCAL=55432 -DTAP_CHECKS=OFF"
+    docker exec -it pgregress /bin/bash -c "cd /timescaledb && ./bootstrap -DCMAKE_BUILD_TYPE=Debug -DTEST_PGHOST=docker.for.win.localhost -DTEST_PGPORT_LOCAL=55432"
 
 test_script:
 - ps: >-

scripts/docker-build.sh

@@ -78,7 +78,7 @@ build_timescaledb()
 
     # Build and install the extension with debug symbols and assertions
     tar -c -C ${BASE_DIR} {cmake,src,sql,test,scripts,tsl,version.config,CMakeLists.txt,timescaledb.control.in} | docker cp - ${BUILD_CONTAINER_NAME}:/build/
-    docker exec -u root ${BUILD_CONTAINER_NAME} /bin/bash -c "cd /build/debug && cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DUSE_OPENSSL=${USE_OPENSSL} -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE=${BUILD_TYPE} -DTAP_CHECKS=OFF /src && make && make install && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample && echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample && cd / && rm -rf /build"
+    docker exec -u root ${BUILD_CONTAINER_NAME} /bin/bash -c "cd /build/debug && cmake -DGENERATE_DOWNGRADE_SCRIPT=${GENERATE_DOWNGRADE_SCRIPT} -DUSE_OPENSSL=${USE_OPENSSL} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} /src && make && make install && echo \"shared_preload_libraries = 'timescaledb'\" >> /usr/local/share/postgresql/postgresql.conf.sample && echo \"timescaledb.telemetry_level=off\" >> /usr/local/share/postgresql/postgresql.conf.sample && cd / && rm -rf /build"
     if [ $? -ne 0 ]; then
       echo "Building timescaledb failed"
       return 1

scripts/docker-run-abi-test.sh

@@ -38,7 +38,7 @@ create_base_container() {
   docker exec -u root $1 /bin/bash -c "apk add --no-cache --virtual .build-deps coreutils dpkg-dev gcc libc-dev make util-linux-dev diffutils cmake bison flex openssl-dev perl perl-utils perl-test-harness-utils perl-ipc-run && mkdir -p /build/debug "
 
   # Copy the source files to build directory
-  docker exec -u root $1 /bin/bash -c "cp -a /src/{src,sql,scripts,test,tsl,cmake,CMakeLists.txt,timescaledb.control.in,version.config} /build/ && cd /build/debug/ && CFLAGS=-Werror cmake .. -DCMAKE_BUILD_TYPE=Debug -DREGRESS_CHECKS=OFF && make -C /build/debug install && chown -R postgres /build"
+  docker exec -u root $1 /bin/bash -c "cp -a /src/{src,sql,scripts,test,tsl,cmake,CMakeLists.txt,timescaledb.control.in,version.config} /build/ && cd /build/debug/ && CFLAGS=-Werror cmake .. -DCMAKE_BUILD_TYPE=Debug && make -C /build/debug install && chown -R postgres /build"
 }
 
 create_base_container $CONTAINER_NAME_COMPILE $PG_IMAGE_TAG_COMPILE

scripts/test_sanitizers.sh

@@ -79,7 +79,7 @@ docker exec -i -u postgres -w /tsdb_build/timescaledb timescaledb-san /bin/bash
 export CFLAGS="-fsanitize=address,undefined -fno-omit-frame-pointer -O2"
 export PG_SOURCE_DIR="/usr/src/postgresql/"
 export BUILD_FORCE_REMOVE=true
-./bootstrap -DREGRESS_CHECKS=OFF -DCMAKE_BUILD_TYPE='Debug' -DTEST_GROUP_SIZE=1
+./bootstrap -DCMAKE_BUILD_TYPE='Debug' -DTEST_GROUP_SIZE=1
 cd build
 make
 EOF

test/CMakeLists.txt

@@ -2,21 +2,16 @@ set(PRIMARY_TEST_DIR ${CMAKE_CURRENT_LIST_DIR})
 set(PRIMARY_TEST_DIR
     ${CMAKE_CURRENT_LIST_DIR}
     PARENT_SCOPE)
+
 set(_local_install_checks)
 set(_install_checks)
 
 # Testing support
-find_program(PG_REGRESS pg_regress
-             HINTS "${PG_BINDIR}" "${PG_PKGLIBDIR}/pgxs/src/test/regress/")
-
-find_program(
-  PG_ISOLATION_REGRESS
-  NAMES pg_isolation_regress
-  HINTS ${PG_BINDIR} ${PG_PKGLIBDIR}/pgxs/src/test/isolation
-        ${PG_SOURCE_DIR}/src/test/isolation ${BINDIR})
-
 include(test-defs.cmake)
 
+# No checks for REGRESS_CHECKS needed here since all the checks are done in the
+# parent CMakeLists.txt.
+
 if(PG_REGRESS)
   message(STATUS "Using pg_regress ${PG_REGRESS}")
 
@@ -41,16 +36,9 @@ if(PG_REGRESS)
 
   list(APPEND _local_install_checks regresschecklocal)
   list(APPEND _install_checks regresscheck)
-elseif(REGRESS_CHECKS)
+elseif(REQUIRE_ALL_TESTS)
   message(
-    FATAL_ERROR
-      "Program 'pg_regress' not found, but regressions checks were required.\n"
-      "Skip regression checks using -DREGRESS_CHECKS=OFF")
-else()
-  message(
-    STATUS
-      "Skipping isolation regress checks since program 'pg_regress' was not found"
-  )
+    FATAL_ERROR "All tests were required but 'pg_regress' could not be found")
 endif()
 
 if(PG_ISOLATION_REGRESS)
@@ -79,21 +67,10 @@ if(PG_ISOLATION_REGRESS)
 
   list(APPEND _local_install_checks isolationchecklocal)
   list(APPEND _install_checks isolationcheck)
-elseif(REGRESS_CHECKS)
+elseif(REQUIRE_ALL_TESTS)
   message(
     FATAL_ERROR
-      "Program 'pg_isolation_regress' not found, but regressions checks were required.\n"
-      "Skip regression checks using -DREGRESS_CHECKS=OFF")
-else()
-  message(
-    STATUS
-      "Skipping isolation regress checks since program 'pg_isolation_regress' was not found"
-  )
-endif()
-
-# installcheck starts up new temporary instances for testing code
-if(NOT APACHE_ONLY)
-  list(APPEND _install_checks installcheck-t)
+      "All tests were required but 'pg_isolation_regress' could not be found")
 endif()
 
 if(_install_checks)

tsl/test/CMakeLists.txt

@@ -5,6 +5,8 @@ add_subdirectory(ssl)
 set(_local_install_checks)
 set(_install_checks)
 
+# No checks for REGRESS_CHECKS needed here since all the checks are done in the
+# parent CMakeLists.txt.
 if(PG_REGRESS)
   add_custom_target(
     regresscheck-t
@@ -51,22 +53,29 @@ if(PG_REGRESS)
       ${PG_REGRESS_OPTS_LOCAL_INSTANCE}
     USES_TERMINAL)
 
-  if(TAP_CHECKS)
-    add_custom_target(
-      provecheck
-      COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/tmp_check
-      COMMAND
-        CONFDIR=${CMAKE_BINARY_DIR}/tsl/test PATH="${PG_BINDIR}:$ENV{PATH}"
-        PG_REGRESS=${PG_REGRESS} SRC_DIR=${PG_SOURCE_DIR}
-        CM_SRC_DIR=${CMAKE_SOURCE_DIR} PG_LIBDIR=${PG_LIBDIR}
-        ${PRIMARY_TEST_DIR}/pg_prove.sh
-      USES_TERMINAL)
-    list(APPEND _install_checks provecheck)
-  endif()
-
   list(APPEND _install_checks regresscheck-shared)
   list(APPEND _local_install_checks regresschecklocal-shared)
+elseif(REQUIRE_ALL_TESTS)
+  message(
+    FATAL_ERROR "All tests were required but 'pg_regress' could not be found")
+endif()
 
+if(TAP_CHECKS)
+  add_custom_target(
+    provecheck
+    COMMAND rm -rf ${CMAKE_CURRENT_BINARY_DIR}/tmp_check
+    COMMAND
+      CONFDIR=${CMAKE_BINARY_DIR}/tsl/test PATH="${PG_BINDIR}:$ENV{PATH}"
+      PG_REGRESS=${PG_REGRESS} SRC_DIR=${PG_SOURCE_DIR}
+      CM_SRC_DIR=${CMAKE_SOURCE_DIR} PG_LIBDIR=${PG_LIBDIR}
+      ${PRIMARY_TEST_DIR}/pg_prove.sh
+    USES_TERMINAL)
+  list(APPEND _install_checks provecheck)
+elseif(REQUIRE_ALL_TESTS)
+  message(
+    FATAL_ERROR
+      "All tests were required but TAP_CHECKS was off (see previous messages why)"
+  )
 endif()
 
 if(PG_ISOLATION_REGRESS)
@@ -94,6 +103,10 @@ if(PG_ISOLATION_REGRESS)
 
   list(APPEND _local_install_checks isolationchecklocal-t)
   list(APPEND _install_checks isolationcheck-t)
+elseif(REQUIRE_ALL_TESTS)
+  message(
+    FATAL_ERROR
+      "All tests were required but 'pg_isolation_regress' could not be found")
 endif()
 
 add_subdirectory(shared)
@@ -102,8 +115,15 @@ add_subdirectory(isolation)
 add_subdirectory(t)
 
 # installchecklocal tests against an existing postgres instance
-add_custom_target(installchecklocal-t DEPENDS ${_local_install_checks})
-add_custom_target(installcheck-t DEPENDS ${_install_checks})
+if(_local_install_checks)
+  add_custom_target(installchecklocal-t DEPENDS ${_local_install_checks})
+  add_dependencies(installchecklocal installchecklocal-t)
+endif()
+
+if(_install_checks)
+  add_custom_target(installcheck-t DEPENDS ${_install_checks})
+  add_dependencies(installcheck installcheck-t)
+endif()
 
 if(CMAKE_BUILD_TYPE MATCHES Debug)
   add_subdirectory(src)