diff --git a/.github/gh_matrix_builder.py b/.github/gh_matrix_builder.py index df912ba6d34..e356e90286f 100644 --- a/.github/gh_matrix_builder.py +++ b/.github/gh_matrix_builder.py @@ -18,170 +18,219 @@ import json import os import sys -from ci_settings import PG12_EARLIEST, PG12_LATEST, PG13_EARLIEST, PG13_LATEST, PG14_EARLIEST, PG14_LATEST +from ci_settings import ( + PG12_EARLIEST, + PG12_LATEST, + PG13_EARLIEST, + PG13_LATEST, + PG14_EARLIEST, + PG14_LATEST, +) # github event type which is either push, pull_request or schedule event_type = sys.argv[1] -m = {"include": [],} +m = { + "include": [], +} # helper functions to generate matrix entries # the release and apache config inherit from the # debug config to reduce repetition def build_debug_config(overrides): - # llvm version and clang versions must match otherwise - # there will be build errors this is true even when compiling - # with gcc as clang is used to compile the llvm parts. - # - # Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but - # we add it as a precaution. Intention is to have at least one - # release and one debug build with WARNINGS_AS_ERRORS=ON so that we - # capture warnings generated due to changes in the code base or the - # compiler. - base_config = dict({ - "name": "Debug", - "build_type": "Debug", - "pg_build_args": "--enable-debug --enable-cassert", - "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", - "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed'", - "coverage": True, - "extra_packages": "clang-9 llvm-9 llvm-9-dev llvm-9-tools", - "llvm_config": "llvm-config-9", - "clang": "clang-9", - "os": "ubuntu-20.04", - "cc": "gcc", - "cxx": "g++", - }) - base_config.update(overrides) - return base_config + # llvm version and clang versions must match otherwise + # there will be build errors this is true even when compiling + # with gcc as clang is used to compile the llvm parts. + # + # Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but + # we add it as a precaution. Intention is to have at least one + # release and one debug build with WARNINGS_AS_ERRORS=ON so that we + # capture warnings generated due to changes in the code base or the + # compiler. + base_config = dict( + { + "name": "Debug", + "build_type": "Debug", + "pg_extra_args": "--enable-debug --enable-cassert", + "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", + "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed'", + "coverage": True, + "extra_packages": "clang-14 llvm-14 llvm-14-dev llvm-14-tools", + "llvm_config": "llvm-config-14", + "clang": "clang-14", + "os": "ubuntu-22.04", + "cc": "gcc", + "cxx": "g++", + } + ) + base_config.update(overrides) + return base_config + # We build this release configuration with WARNINGS_AS_ERRORS=ON to # make sure that we can build with -Werrors even for release # builds. This will capture some cases where warnings are generated # for release builds but not for debug builds. def build_release_config(overrides): - base_config = build_debug_config({}) - release_config = dict({ - "name": "Release", - "build_type": "Release", - "pg_build_args": "", - "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", - "coverage": False, - }) - base_config.update(release_config) - base_config.update(overrides) - return base_config + base_config = build_debug_config({}) + release_config = dict( + { + "name": "Release", + "build_type": "Release", + "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON", + "coverage": False, + } + ) + base_config.update(release_config) + base_config.update(overrides) + return base_config + def build_without_telemetry(overrides): - config = build_release_config({}) - config.update({ - 'name': 'ReleaseWithoutTelemetry', - "tsdb_build_args": config['tsdb_build_args'] + " -DUSE_TELEMETRY=OFF", - "coverage": False, - }) - config.update(overrides) - return config + config = build_release_config({}) + config.update( + { + "name": "ReleaseWithoutTelemetry", + "tsdb_build_args": config["tsdb_build_args"] + " -DUSE_TELEMETRY=OFF", + "coverage": False, + } + ) + config.update(overrides) + return config + def build_apache_config(overrides): - base_config = build_debug_config({}) - apache_config = dict({ - "name": "ApacheOnly", - "build_type": "Release", - "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1", - "pg_build_args": "", - "coverage": False, - }) - base_config.update(apache_config) - base_config.update(overrides) - return base_config + base_config = build_debug_config({}) + apache_config = dict( + { + "name": "ApacheOnly", + "build_type": "Release", + "tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1", + "coverage": False, + } + ) + base_config.update(apache_config) + base_config.update(overrides) + return base_config + def macos_config(overrides): - base_config = dict({ - "pg": PG12_LATEST, - "os": "macos-11", - "cc": "clang", - "cxx": "clang++", - "clang": "clang", - "pg_extra_args": "--with-libraries=/usr/local/opt/openssl/lib --with-includes=/usr/local/opt/openssl/include", - "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl", - "llvm_config": "/usr/local/opt/llvm/bin/llvm-config", - "coverage": False, - "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher pg_dump remote_connection compressed_collation'", - "extra_packages": "", - }) - base_config.update(overrides) - return base_config + base_config = dict( + { + "pg": PG12_LATEST, + "os": "macos-11", + "cc": "clang", + "cxx": "clang++", + # the current github macos image has a buggy llvm installation so we build without llvm on mac + "pg_extra_args": "--with-libraries=/usr/local/opt/openssl/lib --with-includes=/usr/local/opt/openssl/include --without-llvm", + "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl", + "coverage": False, + "installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher pg_dump remote_connection compressed_collation'", + "extra_packages": "", + } + ) + base_config.update(overrides) + return base_config + # always test debug build on latest of all supported pg versions -m["include"].append(build_debug_config({"pg":PG12_LATEST})) -m["include"].append(build_debug_config({"pg":PG13_LATEST})) -m["include"].append(build_debug_config({"pg":PG14_LATEST})) +m["include"].append(build_debug_config({"pg": PG12_LATEST})) +m["include"].append( + build_debug_config({"pg": PG13_LATEST, "cc": "clang-14", "cxx": "clang++-14"}) +) +m["include"].append(build_debug_config({"pg": PG14_LATEST})) m["include"].append(build_release_config(macos_config({}))) -m["include"].append(build_without_telemetry({"pg":PG14_LATEST})) - -m["include"].append(build_debug_config({"pg":15,"snapshot":"snapshot", -"tsdb_build_args":"-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DEXPERIMENTAL=ON -DCODECOVERAGE=ON", -# below tests are tracked as part of #4838 - "installcheck_args": "SKIPS='003_connections_privs 001_simple_multinode 004_multinode_rdwr_1pc data_node_bootstrap dist_hypertable-15 bgw_custom' " -# below tests are tracked as part of #4832 - "IGNORES='pg_dump_unprivileged cagg_dump metadata pg_dump compression_hypertable " -# below tests are tracked as part of #4833 - "telemetry_stats dist_commands dist_ddl dist_query dist_partial_agg continuous_aggs continuous_aggs_deprecated " -# below tests are tracked as part of #4834 - "compression compression_dml modify_exclusion rowsecurity-15 " -# below tests are tracked as part of #4835 - "parallel plan_hashagg query partialize_finalize dist_distinct dist_fetcher_type dist_remote_error jit-15 " -# below tests are tracked as part of #4837 - "remote_txn'"})) +m["include"].append(build_without_telemetry({"pg": PG14_LATEST})) + +m["include"].append( + build_debug_config( + { + "pg": 15, + "snapshot": "snapshot", + "tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DEXPERIMENTAL=ON -DCODECOVERAGE=ON", + # below tests are tracked as part of #4838 + "installcheck_args": "SKIPS='003_connections_privs 001_simple_multinode 004_multinode_rdwr_1pc data_node_bootstrap dist_hypertable-15 bgw_custom' " + # below tests are tracked as part of #4832 + "IGNORES='pg_dump_unprivileged cagg_dump metadata pg_dump compression_hypertable " + # below tests are tracked as part of #4833 + "telemetry_stats dist_commands dist_ddl dist_query dist_partial_agg continuous_aggs continuous_aggs_deprecated " + # below tests are tracked as part of #4834 + "compression compression_dml modify_exclusion rowsecurity-15 " + # below tests are tracked as part of #4835 + "parallel plan_hashagg query partialize_finalize dist_distinct dist_fetcher_type dist_remote_error jit-15 " + # below tests are tracked as part of #4837 + "remote_txn'", + } + ) +) # if this is not a pull request e.g. a scheduled run or a push # to a specific branch like prerelease_test we add additional # entries to the matrix if event_type != "pull_request": - - # add debug test for first supported PG12 version - # most of the IGNORES are the isolation tests because the output format has changed between versions - # chunk_utils, telemetry and tablespace are skipped because of use after free bugs in postgres 12.0 which those tests hit - pg12_debug_earliest = { - "pg": PG12_EARLIEST, - "installcheck_args": "SKIPS='chunk_utils tablespace telemetry' IGNORES='cluster-12 cagg_policy debug_notice dist_gapfill_pushdown-12'", - "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF", - } - m["include"].append(build_debug_config(pg12_debug_earliest)) - - # add debug test for first supported PG13 version - pg13_debug_earliest = { - "pg": PG13_EARLIEST, - "installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'", - "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF", - } - m["include"].append(build_debug_config(pg13_debug_earliest)) - - # add debug test for first supported PG14 version - m["include"].append(build_debug_config({"pg": PG14_EARLIEST, "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'"})) - - # add debug test for MacOS - m["include"].append(build_debug_config(macos_config({}))) - - # add release test for latest pg 12 and 13 - m["include"].append(build_release_config({"pg":PG12_LATEST})) - m["include"].append(build_release_config({"pg":PG13_LATEST})) - m["include"].append(build_release_config({"pg":PG14_LATEST})) - - # add apache only test for latest pg - m["include"].append(build_apache_config({"pg":PG12_LATEST})) - m["include"].append(build_apache_config({"pg":PG13_LATEST})) - m["include"].append(build_apache_config({"pg":PG14_LATEST})) - - # to discover issues with upcoming releases we run CI against - # the stable branches of supported PG releases - m["include"].append(build_debug_config({"pg":12,"snapshot":"snapshot"})) - m["include"].append(build_debug_config({"pg":13,"snapshot":"snapshot"})) - m["include"].append(build_debug_config({"pg":14,"snapshot":"snapshot", "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'"})) + # add debug test for first supported PG12 version + # most of the IGNORES are the isolation tests because the output format has changed between versions + # chunk_utils, telemetry and tablespace are skipped because of use after free bugs in postgres 12.0 which those tests hit + pg12_debug_earliest = { + "pg": PG12_EARLIEST, + # The early releases don't build with llvm 14. + "pg_extra_args": "--enable-debug --enable-cassert --without-llvm", + "installcheck_args": "SKIPS='chunk_utils tablespace telemetry' IGNORES='cluster-12 cagg_policy debug_notice dist_gapfill_pushdown-12'", + "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF", + } + m["include"].append(build_debug_config(pg12_debug_earliest)) + + # add debug test for first supported PG13 version + pg13_debug_earliest = { + "pg": PG13_EARLIEST, + # The early releases don't build with llvm 14. + "pg_extra_args": "--enable-debug --enable-cassert --without-llvm", + "installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'", + "tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF", + } + m["include"].append(build_debug_config(pg13_debug_earliest)) + + # add debug test for first supported PG14 version + m["include"].append( + build_debug_config( + { + "pg": PG14_EARLIEST, + # The early releases don't build with llvm 14. + "pg_extra_args": "--enable-debug --enable-cassert --without-llvm", + "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'", + } + ) + ) + + # add debug test for MacOS + m["include"].append(build_debug_config(macos_config({}))) + + # add release test for latest pg 12 and 13 + m["include"].append(build_release_config({"pg": PG12_LATEST})) + m["include"].append(build_release_config({"pg": PG13_LATEST})) + m["include"].append(build_release_config({"pg": PG14_LATEST})) + + # add apache only test for latest pg + m["include"].append(build_apache_config({"pg": PG12_LATEST})) + m["include"].append(build_apache_config({"pg": PG13_LATEST})) + m["include"].append(build_apache_config({"pg": PG14_LATEST})) + + # to discover issues with upcoming releases we run CI against + # the stable branches of supported PG releases + m["include"].append(build_debug_config({"pg": 12, "snapshot": "snapshot"})) + m["include"].append(build_debug_config({"pg": 13, "snapshot": "snapshot"})) + m["include"].append( + build_debug_config( + { + "pg": 14, + "snapshot": "snapshot", + "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'", + } + ) + ) # generate command to set github action variable -with open(os.environ['GITHUB_OUTPUT'], "a") as output: - print(str.format("matrix={0}",json.dumps(m)), file=output) - +with open(os.environ["GITHUB_OUTPUT"], "a") as output: + print(str.format("matrix={0}", json.dumps(m)), file=output) diff --git a/.github/workflows/linux-build-and-test.yaml b/.github/workflows/linux-build-and-test.yaml index c68af509564..4e79b929ba0 100644 --- a/.github/workflows/linux-build-and-test.yaml +++ b/.github/workflows/linux-build-and-test.yaml @@ -38,8 +38,12 @@ jobs: - name: Install Linux Dependencies if: runner.os == 'Linux' run: | + # Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable. + # If needed, install them before opening the core dump. sudo apt-get update sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl libtest-most-perl ${{ matrix.extra_packages }} + # The GCC 11 gcov segfaults with the coverage info generated by clang. + sudo ln -sf $(which llvm-cov-14) $(which gcov) - name: Install macOS Dependencies if: runner.os == 'macOS' @@ -80,12 +84,7 @@ jobs: mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 cd ~/$PG_SRC_DIR - if [[ "${{ runner.os }}" == "Linux" ]]; then - ./configure --prefix=$HOME/$PG_INSTALL_DIR ${{ matrix.pg_build_args }} --with-llvm LLVM_CONFIG=${{ matrix.llvm_config }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }} - else - # the current github macos image has a buggy llvm installation so we build without llvm on mac - ./configure --prefix=$HOME/$PG_INSTALL_DIR ${{ matrix.pg_build_args }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }} - fi + ./configure --prefix=$HOME/$PG_INSTALL_DIR --with-llvm LLVM_CONFIG=${{ matrix.llvm_config }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }} make -j $MAKE_JOBS make -j $MAKE_JOBS -C src/test/isolation make -j $MAKE_JOBS -C contrib/postgres_fdw @@ -169,6 +168,8 @@ jobs: if: always() && steps.collectlogs.outputs.coredumps == 'true' run: | sudo coredumpctl gdb <fd.application_name)); MemoryContext oldcontext = CurrentMemoryContext; TimestampTz start_time = DT_NOBEGIN, finish_time = DT_NOBEGIN; - NameData proc_schema = { 0 }, proc_name = { 0 }; + NameData proc_schema = { .data = { 0 } }, proc_name = { .data = { 0 } }; PG_TRY(); { diff --git a/src/bgw/job_stat.c b/src/bgw/job_stat.c index 11427555c3c..6fc8a6098ba 100644 --- a/src/bgw/job_stat.c +++ b/src/bgw/job_stat.c @@ -699,7 +699,7 @@ ts_bgw_job_stat_next_start(BgwJobStat *jobstat, BgwJob *job, int32 consecutive_f if (!ts_flags_are_set_32(jobstat->fd.flags, LAST_CRASH_REPORTED)) { /* add the proc_schema, proc_name to the jsonb */ - NameData proc_schema = { 0 }, proc_name = { 0 }; + NameData proc_schema = { .data = { 0 } }, proc_name = { .data = { 0 } }; namestrcpy(&proc_schema, NameStr(job->fd.proc_schema)); namestrcpy(&proc_name, NameStr(job->fd.proc_name)); JsonbParseState *parse_state = NULL; diff --git a/src/chunk.c b/src/chunk.c index c80ce0209e0..2c1a702a171 100644 --- a/src/chunk.c +++ b/src/chunk.c @@ -1911,7 +1911,7 @@ chunk_resurrect(const Hypertable *ht, int chunk_id) { ScanIterator iterator; Chunk *chunk = NULL; - int count = 0; + PG_USED_FOR_ASSERTS_ONLY int count = 0; Assert(chunk_id != 0); diff --git a/src/hypertable.c b/src/hypertable.c index 247b321a507..691e9905ec9 100644 --- a/src/hypertable.c +++ b/src/hypertable.c @@ -466,8 +466,8 @@ ts_hypertable_scan_with_memory_context(const char *schema, const char *table, bool tuplock, MemoryContext mctx) { ScanKeyData scankey[2]; - NameData schema_name = { { 0 } }; - NameData table_name = { { 0 } }; + NameData schema_name = { .data = { 0 } }; + NameData table_name = { .data = { 0 } }; if (schema) namestrcpy(&schema_name, schema); diff --git a/src/ts_catalog/continuous_agg.c b/src/ts_catalog/continuous_agg.c index 6960e908be0..6d0db3d4808 100644 --- a/src/ts_catalog/continuous_agg.c +++ b/src/ts_catalog/continuous_agg.c @@ -1304,7 +1304,7 @@ ts_number_of_continuous_aggs() static int32 find_raw_hypertable_for_materialization(int32 mat_hypertable_id) { - short count = 0; + PG_USED_FOR_ASSERTS_ONLY short count = 0; int32 htid = INVALID_HYPERTABLE_ID; ScanIterator iterator = ts_scan_iterator_create(CONTINUOUS_AGG, RowExclusiveLock, CurrentMemoryContext); diff --git a/tsl/src/bgw_policy/job_api.c b/tsl/src/bgw_policy/job_api.c index 3a4f303c031..d18fb734f1c 100644 --- a/tsl/src/bgw_policy/job_api.c +++ b/tsl/src/bgw_policy/job_api.c @@ -40,8 +40,8 @@ validate_check_signature(Oid check) { Oid proc = InvalidOid; ObjectWithArgs *object; - NameData check_name = { 0 }; - NameData check_schema = { 0 }; + NameData check_name = { .data = { 0 } }; + NameData check_schema = { .data = { 0 } }; namestrcpy(&check_schema, get_namespace_name(get_func_namespace(check))); namestrcpy(&check_name, get_func_name(check)); @@ -80,8 +80,8 @@ job_add(PG_FUNCTION_ARGS) NameData proc_name; NameData proc_schema; NameData owner_name; - NameData check_name = { 0 }; - NameData check_schema = { 0 }; + NameData check_name = { .data = { 0 } }; + NameData check_schema = { .data = { 0 } }; Interval max_runtime = { .time = DEFAULT_MAX_RUNTIME }; Interval retry_period = { .time = DEFAULT_RETRY_PERIOD }; int32 job_id; @@ -293,8 +293,8 @@ job_alter(PG_FUNCTION_ARGS) int job_id = PG_GETARG_INT32(0); bool if_exists = PG_GETARG_BOOL(8); BgwJob *job; - NameData check_name = { 0 }; - NameData check_schema = { 0 }; + NameData check_name = { .data = { 0 } }; + NameData check_schema = { .data = { 0 } }; Oid check = PG_ARGISNULL(9) ? InvalidOid : PG_GETARG_OID(9); char *check_name_str = NULL; /* Added space for period and NULL */ @@ -370,7 +370,7 @@ job_alter(PG_FUNCTION_ARGS) if (unregister_check) { - NameData empty_namedata = { 0 }; + NameData empty_namedata = { .data = { 0 } }; namestrcpy(&job->fd.check_schema, NameStr(empty_namedata)); namestrcpy(&job->fd.check_name, NameStr(empty_namedata)); } diff --git a/tsl/src/compression/create.c b/tsl/src/compression/create.c index 5d435447251..5449a0cc00d 100644 --- a/tsl/src/compression/create.c +++ b/tsl/src/compression/create.c @@ -918,7 +918,7 @@ check_modify_compression_options(Hypertable *ht, WithClauseResult *with_clause_o bool orderby_time_default_matches = false; ListCell *elem1, *elem2; FormData_hypertable_compression *fd_elem1; - NameData colname1 = { { 0 } }, colname2 = { { 0 } }; + NameData colname1 = { .data = { 0 } }, colname2 = { .data = { 0 } }; CompressedParsedCol *cpc_elem2; /* If the orderby that's already set is only the time column DESC (which is the default), and we pass the default again, then no need to give an error */