Build on Ubuntu 22.04

It has newer GCC which should detect more warnings.
timescale · Oct 26, 2022 · 864da20 · 864da20
1 parent 2e7d7ee
commit 864da20
Show file tree

Hide file tree

Showing 10 changed files with 207 additions and 155 deletions.
diff --git a/.github/gh_matrix_builder.py b/.github/gh_matrix_builder.py
@@ -18,170 +18,219 @@
 import json
 import os
 import sys
-from ci_settings import PG12_EARLIEST, PG12_LATEST, PG13_EARLIEST, PG13_LATEST, PG14_EARLIEST, PG14_LATEST
+from ci_settings import (
+    PG12_EARLIEST,
+    PG12_LATEST,
+    PG13_EARLIEST,
+    PG13_LATEST,
+    PG14_EARLIEST,
+    PG14_LATEST,
+)
 
 # github event type which is either push, pull_request or schedule
 event_type = sys.argv[1]
 
-m = {"include": [],}
+m = {
+    "include": [],
+}
 
 # helper functions to generate matrix entries
 # the release and apache config inherit from the
 # debug config to reduce repetition
 def build_debug_config(overrides):
-  # llvm version and clang versions must match otherwise
-  # there will be build errors this is true even when compiling
-  # with gcc as clang is used to compile the llvm parts.
-  #
-  # Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but
-  # we add it as a precaution. Intention is to have at least one
-  # release and one debug build with WARNINGS_AS_ERRORS=ON so that we
-  # capture warnings generated due to changes in the code base or the
-  # compiler.
-  base_config = dict({
-    "name": "Debug",
-    "build_type": "Debug",
-    "pg_build_args": "--enable-debug --enable-cassert",
-    "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
-    "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed'",
-    "coverage": True,
-    "extra_packages": "clang-9 llvm-9 llvm-9-dev llvm-9-tools",
-    "llvm_config": "llvm-config-9",
-    "clang": "clang-9",
-    "os": "ubuntu-20.04",
-    "cc": "gcc",
-    "cxx": "g++",
-  })
-  base_config.update(overrides)
-  return base_config
+    # llvm version and clang versions must match otherwise
+    # there will be build errors this is true even when compiling
+    # with gcc as clang is used to compile the llvm parts.
+    #
+    # Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but
+    # we add it as a precaution. Intention is to have at least one
+    # release and one debug build with WARNINGS_AS_ERRORS=ON so that we
+    # capture warnings generated due to changes in the code base or the
+    # compiler.
+    base_config = dict(
+        {
+            "name": "Debug",
+            "build_type": "Debug",
+            "pg_extra_args": "--enable-debug --enable-cassert",
+            "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
+            "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed'",
+            "coverage": True,
+            "extra_packages": "clang-14 llvm-14 llvm-14-dev llvm-14-tools",
+            "llvm_config": "llvm-config-14",
+            "clang": "clang-14",
+            "os": "ubuntu-22.04",
+            "cc": "gcc",
+            "cxx": "g++",
+        }
+    )
+    base_config.update(overrides)
+    return base_config
+
 
 # We build this release configuration with WARNINGS_AS_ERRORS=ON to
 # make sure that we can build with -Werrors even for release
 # builds. This will capture some cases where warnings are generated
 # for release builds but not for debug builds.
 def build_release_config(overrides):
-  base_config = build_debug_config({})
-  release_config = dict({
-    "name": "Release",
-    "build_type": "Release",
-    "pg_build_args": "",
-    "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
-    "coverage": False,
-  })
-  base_config.update(release_config)
-  base_config.update(overrides)
-  return base_config
+    base_config = build_debug_config({})
+    release_config = dict(
+        {
+            "name": "Release",
+            "build_type": "Release",
+            "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
+            "coverage": False,
+        }
+    )
+    base_config.update(release_config)
+    base_config.update(overrides)
+    return base_config
+
 
 def build_without_telemetry(overrides):
-  config = build_release_config({})
-  config.update({
-    'name': 'ReleaseWithoutTelemetry',
-    "tsdb_build_args": config['tsdb_build_args'] + " -DUSE_TELEMETRY=OFF",
-    "coverage": False,
-  })
-  config.update(overrides)
-  return config
+    config = build_release_config({})
+    config.update(
+        {
+            "name": "ReleaseWithoutTelemetry",
+            "tsdb_build_args": config["tsdb_build_args"] + " -DUSE_TELEMETRY=OFF",
+            "coverage": False,
+        }
+    )
+    config.update(overrides)
+    return config
+
 
 def build_apache_config(overrides):
-  base_config = build_debug_config({})
-  apache_config = dict({
-    "name": "ApacheOnly",
-    "build_type": "Release",
-    "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1",
-    "pg_build_args": "",
-    "coverage": False,
-  })
-  base_config.update(apache_config)
-  base_config.update(overrides)
-  return base_config
+    base_config = build_debug_config({})
+    apache_config = dict(
+        {
+            "name": "ApacheOnly",
+            "build_type": "Release",
+            "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1",
+            "coverage": False,
+        }
+    )
+    base_config.update(apache_config)
+    base_config.update(overrides)
+    return base_config
+
 
 def macos_config(overrides):
-  base_config = dict({
-    "pg": PG12_LATEST,
-    "os": "macos-11",
-    "cc": "clang",
-    "cxx": "clang++",
-    "clang": "clang",
-    "pg_extra_args": "--with-libraries=/usr/local/opt/openssl/lib --with-includes=/usr/local/opt/openssl/include",
-    "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl",
-    "llvm_config": "/usr/local/opt/llvm/bin/llvm-config",
-    "coverage": False,
-    "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher pg_dump remote_connection compressed_collation'",
-    "extra_packages": "",
-  })
-  base_config.update(overrides)
-  return base_config
+    base_config = dict(
+        {
+            "pg": PG12_LATEST,
+            "os": "macos-11",
+            "cc": "clang",
+            "cxx": "clang++",
+            # the current github macos image has a buggy llvm installation so we build without llvm on mac
+            "pg_extra_args": "--with-libraries=/usr/local/opt/openssl/lib --with-includes=/usr/local/opt/openssl/include --without-llvm",
+            "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl",
+            "coverage": False,
+            "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher pg_dump remote_connection compressed_collation'",
+            "extra_packages": "",
+        }
+    )
+    base_config.update(overrides)
+    return base_config
+
 
 # always test debug build on latest of all supported pg versions
-m["include"].append(build_debug_config({"pg":PG12_LATEST}))
-m["include"].append(build_debug_config({"pg":PG13_LATEST}))
-m["include"].append(build_debug_config({"pg":PG14_LATEST}))
+m["include"].append(build_debug_config({"pg": PG12_LATEST}))
+m["include"].append(
+    build_debug_config({"pg": PG13_LATEST, "cc": "clang-14", "cxx": "clang++-14"})
+)
+m["include"].append(build_debug_config({"pg": PG14_LATEST}))
 
 m["include"].append(build_release_config(macos_config({})))
 
-m["include"].append(build_without_telemetry({"pg":PG14_LATEST}))
-
-m["include"].append(build_debug_config({"pg":15,"snapshot":"snapshot", 
-"tsdb_build_args":"-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DEXPERIMENTAL=ON -DCODECOVERAGE=ON",
-# below tests are tracked as part of #4838
-      "installcheck_args": "SKIPS='003_connections_privs 001_simple_multinode 004_multinode_rdwr_1pc data_node_bootstrap dist_hypertable-15 bgw_custom' "
-# below tests are tracked as part of #4832
-      "IGNORES='pg_dump_unprivileged cagg_dump metadata pg_dump compression_hypertable "
-# below tests are tracked as part of #4833
-      "telemetry_stats dist_commands dist_ddl dist_query dist_partial_agg continuous_aggs continuous_aggs_deprecated "
-# below tests are tracked as part of #4834
-      "compression compression_dml modify_exclusion rowsecurity-15 "
-# below tests are tracked as part of #4835
-      "parallel plan_hashagg query  partialize_finalize dist_distinct dist_fetcher_type dist_remote_error jit-15 "
-# below tests are tracked as part of #4837
-      "remote_txn'"}))
+m["include"].append(build_without_telemetry({"pg": PG14_LATEST}))
+
+m["include"].append(
+    build_debug_config(
+        {
+            "pg": 15,
+            "snapshot": "snapshot",
+            "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DEXPERIMENTAL=ON -DCODECOVERAGE=ON",
+            # below tests are tracked as part of #4838
+            "installcheck_args": "SKIPS='003_connections_privs 001_simple_multinode 004_multinode_rdwr_1pc data_node_bootstrap dist_hypertable-15 bgw_custom' "
+            # below tests are tracked as part of #4832
+            "IGNORES='pg_dump_unprivileged cagg_dump metadata pg_dump compression_hypertable "
+            # below tests are tracked as part of #4833
+            "telemetry_stats dist_commands dist_ddl dist_query dist_partial_agg continuous_aggs continuous_aggs_deprecated "
+            # below tests are tracked as part of #4834
+            "compression compression_dml modify_exclusion rowsecurity-15 "
+            # below tests are tracked as part of #4835
+            "parallel plan_hashagg query  partialize_finalize dist_distinct dist_fetcher_type dist_remote_error jit-15 "
+            # below tests are tracked as part of #4837
+            "remote_txn'",
+        }
+    )
+)
 
 # if this is not a pull request e.g. a scheduled run or a push
 # to a specific branch like prerelease_test we add additional
 # entries to the matrix
 if event_type != "pull_request":
-
-  # add debug test for first supported PG12 version
-  # most of the IGNORES are the isolation tests because the output format has changed between versions
-  # chunk_utils, telemetry and tablespace are skipped because of use after free bugs in postgres 12.0 which those tests hit
-  pg12_debug_earliest = {
-    "pg": PG12_EARLIEST,
-    "installcheck_args": "SKIPS='chunk_utils tablespace telemetry' IGNORES='cluster-12 cagg_policy debug_notice dist_gapfill_pushdown-12'",
-    "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
-  }
-  m["include"].append(build_debug_config(pg12_debug_earliest))
-
-  # add debug test for first supported PG13 version
-  pg13_debug_earliest = {
-    "pg": PG13_EARLIEST,
-    "installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'",
-    "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
-  }
-  m["include"].append(build_debug_config(pg13_debug_earliest))
-
-  # add debug test for first supported PG14 version
-  m["include"].append(build_debug_config({"pg": PG14_EARLIEST, "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'"}))
-
-  # add debug test for MacOS
-  m["include"].append(build_debug_config(macos_config({})))
-
-  # add release test for latest pg 12 and 13
-  m["include"].append(build_release_config({"pg":PG12_LATEST}))
-  m["include"].append(build_release_config({"pg":PG13_LATEST}))
-  m["include"].append(build_release_config({"pg":PG14_LATEST}))
-
-  # add apache only test for latest pg
-  m["include"].append(build_apache_config({"pg":PG12_LATEST}))
-  m["include"].append(build_apache_config({"pg":PG13_LATEST}))
-  m["include"].append(build_apache_config({"pg":PG14_LATEST}))
-
-  # to discover issues with upcoming releases we run CI against
-  # the stable branches of supported PG releases
-  m["include"].append(build_debug_config({"pg":12,"snapshot":"snapshot"}))
-  m["include"].append(build_debug_config({"pg":13,"snapshot":"snapshot"}))
-  m["include"].append(build_debug_config({"pg":14,"snapshot":"snapshot", "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'"}))
+    # add debug test for first supported PG12 version
+    # most of the IGNORES are the isolation tests because the output format has changed between versions
+    # chunk_utils, telemetry and tablespace are skipped because of use after free bugs in postgres 12.0 which those tests hit
+    pg12_debug_earliest = {
+        "pg": PG12_EARLIEST,
+        # The early releases don't build with llvm 14.
+        "pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
+        "installcheck_args": "SKIPS='chunk_utils tablespace telemetry' IGNORES='cluster-12 cagg_policy debug_notice dist_gapfill_pushdown-12'",
+        "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
+    }
+    m["include"].append(build_debug_config(pg12_debug_earliest))
+
+    # add debug test for first supported PG13 version
+    pg13_debug_earliest = {
+        "pg": PG13_EARLIEST,
+        # The early releases don't build with llvm 14.
+        "pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
+        "installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'",
+        "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
+    }
+    m["include"].append(build_debug_config(pg13_debug_earliest))
+
+    # add debug test for first supported PG14 version
+    m["include"].append(
+        build_debug_config(
+            {
+                "pg": PG14_EARLIEST,
+                # The early releases don't build with llvm 14.
+                "pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
+                "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'",
+            }
+        )
+    )
+
+    # add debug test for MacOS
+    m["include"].append(build_debug_config(macos_config({})))
+
+    # add release test for latest pg 12 and 13
+    m["include"].append(build_release_config({"pg": PG12_LATEST}))
+    m["include"].append(build_release_config({"pg": PG13_LATEST}))
+    m["include"].append(build_release_config({"pg": PG14_LATEST}))
+
+    # add apache only test for latest pg
+    m["include"].append(build_apache_config({"pg": PG12_LATEST}))
+    m["include"].append(build_apache_config({"pg": PG13_LATEST}))
+    m["include"].append(build_apache_config({"pg": PG14_LATEST}))
+
+    # to discover issues with upcoming releases we run CI against
+    # the stable branches of supported PG releases
+    m["include"].append(build_debug_config({"pg": 12, "snapshot": "snapshot"}))
+    m["include"].append(build_debug_config({"pg": 13, "snapshot": "snapshot"}))
+    m["include"].append(
+        build_debug_config(
+            {
+                "pg": 14,
+                "snapshot": "snapshot",
+                "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'",
+            }
+        )
+    )
 
 # generate command to set github action variable
-with open(os.environ['GITHUB_OUTPUT'], "a") as output:
-  print(str.format("matrix={0}",json.dumps(m)), file=output)
-
+with open(os.environ["GITHUB_OUTPUT"], "a") as output:
+    print(str.format("matrix={0}", json.dumps(m)), file=output)
diff --git a/.github/workflows/linux-build-and-test.yaml b/.github/workflows/linux-build-and-test.yaml
@@ -38,8 +38,12 @@ jobs:
     - name: Install Linux Dependencies
       if: runner.os == 'Linux'
       run: |
+        # Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable.
+        # If needed, install them before opening the core dump.
         sudo apt-get update
         sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl libtest-most-perl ${{ matrix.extra_packages }}
+        # The GCC 11 gcov segfaults with the coverage info generated by clang.
+        sudo ln -sf $(which llvm-cov-14) $(which gcov)
 
     - name: Install macOS Dependencies
       if: runner.os == 'macOS'
@@ -80,12 +84,7 @@ jobs:
         mkdir -p ~/$PG_SRC_DIR
         tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1
         cd ~/$PG_SRC_DIR
-        if [[ "${{ runner.os }}" == "Linux" ]]; then
-          ./configure --prefix=$HOME/$PG_INSTALL_DIR ${{ matrix.pg_build_args }} --with-llvm LLVM_CONFIG=${{ matrix.llvm_config }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }}
-        else
-          # the current github macos image has a buggy llvm installation so we build without llvm on mac
-          ./configure --prefix=$HOME/$PG_INSTALL_DIR ${{ matrix.pg_build_args }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }}
-        fi
+        ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-llvm LLVM_CONFIG=${{ matrix.llvm_config }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }}
         make -j $MAKE_JOBS
         make -j $MAKE_JOBS -C src/test/isolation
         make -j $MAKE_JOBS -C contrib/postgres_fdw
@@ -169,6 +168,8 @@ jobs:
       if: always() && steps.collectlogs.outputs.coredumps == 'true'
       run: |
         sudo coredumpctl gdb <<EOT
+          set verbose on
+          show debug-file-directory
           printf "%s\n\n", debug_query_string
           bt full
         EOT