Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Build on Ubuntu 22.04 #4867

Merged
merged 1 commit into from Oct 26, 2022
Merged

Build on Ubuntu 22.04 #4867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
317 changes: 183 additions & 134 deletions .github/gh_matrix_builder.py
View file
Open in desktop
Expand Up @@ -18,170 +18,219 @@
import json
import os
import sys
from ci_settings import PG12_EARLIEST, PG12_LATEST, PG13_EARLIEST, PG13_LATEST, PG14_EARLIEST, PG14_LATEST
from ci_settings import (
PG12_EARLIEST,
PG12_LATEST,
PG13_EARLIEST,
PG13_LATEST,
PG14_EARLIEST,
PG14_LATEST,
)

# github event type which is either push, pull_request or schedule
event_type = sys.argv[1]

m = {"include": [],}
m = {
"include": [],
}

# helper functions to generate matrix entries
# the release and apache config inherit from the
# debug config to reduce repetition
def build_debug_config(overrides):
# llvm version and clang versions must match otherwise
# there will be build errors this is true even when compiling
# with gcc as clang is used to compile the llvm parts.
#
# Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but
# we add it as a precaution. Intention is to have at least one
# release and one debug build with WARNINGS_AS_ERRORS=ON so that we
# capture warnings generated due to changes in the code base or the
# compiler.
base_config = dict({
"name": "Debug",
"build_type": "Debug",
"pg_build_args": "--enable-debug --enable-cassert",
"tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
"installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed'",
"coverage": True,
"extra_packages": "clang-9 llvm-9 llvm-9-dev llvm-9-tools",
"llvm_config": "llvm-config-9",
"clang": "clang-9",
"os": "ubuntu-20.04",
"cc": "gcc",
"cxx": "g++",
})
base_config.update(overrides)
return base_config
# llvm version and clang versions must match otherwise
# there will be build errors this is true even when compiling
# with gcc as clang is used to compile the llvm parts.
#
# Strictly speaking, WARNINGS_AS_ERRORS=ON is not needed here, but
# we add it as a precaution. Intention is to have at least one
# release and one debug build with WARNINGS_AS_ERRORS=ON so that we
# capture warnings generated due to changes in the code base or the
# compiler.
base_config = dict(
{
"name": "Debug",
"build_type": "Debug",
"pg_extra_args": "--enable-debug --enable-cassert",
"tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
"installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed'",
"coverage": True,
"extra_packages": "clang-14 llvm-14 llvm-14-dev llvm-14-tools",
"llvm_config": "llvm-config-14",
"clang": "clang-14",
"os": "ubuntu-22.04",
"cc": "gcc",
"cxx": "g++",
}
)
base_config.update(overrides)
return base_config


# We build this release configuration with WARNINGS_AS_ERRORS=ON to
# make sure that we can build with -Werrors even for release
# builds. This will capture some cases where warnings are generated
# for release builds but not for debug builds.
def build_release_config(overrides):
base_config = build_debug_config({})
release_config = dict({
"name": "Release",
"build_type": "Release",
"pg_build_args": "",
"tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
"coverage": False,
})
base_config.update(release_config)
base_config.update(overrides)
return base_config
base_config = build_debug_config({})
release_config = dict(
{
"name": "Release",
"build_type": "Release",
"tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON",
"coverage": False,
}
)
base_config.update(release_config)
base_config.update(overrides)
return base_config


def build_without_telemetry(overrides):
config = build_release_config({})
config.update({
'name': 'ReleaseWithoutTelemetry',
"tsdb_build_args": config['tsdb_build_args'] + " -DUSE_TELEMETRY=OFF",
"coverage": False,
})
config.update(overrides)
return config
config = build_release_config({})
config.update(
{
"name": "ReleaseWithoutTelemetry",
"tsdb_build_args": config["tsdb_build_args"] + " -DUSE_TELEMETRY=OFF",
"coverage": False,
}
)
config.update(overrides)
return config


def build_apache_config(overrides):
base_config = build_debug_config({})
apache_config = dict({
"name": "ApacheOnly",
"build_type": "Release",
"tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1",
"pg_build_args": "",
"coverage": False,
})
base_config.update(apache_config)
base_config.update(overrides)
return base_config
base_config = build_debug_config({})
apache_config = dict(
{
"name": "ApacheOnly",
"build_type": "Release",
"tsdb_build_args": "-DWARNINGS_AS_ERRORS=ON -DREQUIRE_ALL_TESTS=ON -DAPACHE_ONLY=1",
"coverage": False,
}
)
base_config.update(apache_config)
base_config.update(overrides)
return base_config


def macos_config(overrides):
base_config = dict({
"pg": PG12_LATEST,
"os": "macos-11",
"cc": "clang",
"cxx": "clang++",
"clang": "clang",
"pg_extra_args": "--with-libraries=/usr/local/opt/openssl/lib --with-includes=/usr/local/opt/openssl/include",
"tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl",
"llvm_config": "/usr/local/opt/llvm/bin/llvm-config",
"coverage": False,
"installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher pg_dump remote_connection compressed_collation'",
"extra_packages": "",
})
base_config.update(overrides)
return base_config
base_config = dict(
{
"pg": PG12_LATEST,
"os": "macos-11",
"cc": "clang",
"cxx": "clang++",
# the current github macos image has a buggy llvm installation so we build without llvm on mac
"pg_extra_args": "--with-libraries=/usr/local/opt/openssl/lib --with-includes=/usr/local/opt/openssl/include --without-llvm",
"tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DOPENSSL_ROOT_DIR=/usr/local/opt/openssl",
"coverage": False,
"installcheck_args": "IGNORES='bgw_db_scheduler bgw_db_scheduler_fixed bgw_launcher pg_dump remote_connection compressed_collation'",
"extra_packages": "",
}
)
base_config.update(overrides)
return base_config


# always test debug build on latest of all supported pg versions
m["include"].append(build_debug_config({"pg":PG12_LATEST}))
m["include"].append(build_debug_config({"pg":PG13_LATEST}))
m["include"].append(build_debug_config({"pg":PG14_LATEST}))
m["include"].append(build_debug_config({"pg": PG12_LATEST}))
m["include"].append(
build_debug_config({"pg": PG13_LATEST, "cc": "clang-14", "cxx": "clang++-14"})
)
m["include"].append(build_debug_config({"pg": PG14_LATEST}))

m["include"].append(build_release_config(macos_config({})))

m["include"].append(build_without_telemetry({"pg":PG14_LATEST}))

m["include"].append(build_debug_config({"pg":15,"snapshot":"snapshot",
"tsdb_build_args":"-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DEXPERIMENTAL=ON -DCODECOVERAGE=ON",
# below tests are tracked as part of #4838
"installcheck_args": "SKIPS='003_connections_privs 001_simple_multinode 004_multinode_rdwr_1pc data_node_bootstrap dist_hypertable-15 bgw_custom' "
# below tests are tracked as part of #4832
"IGNORES='pg_dump_unprivileged cagg_dump metadata pg_dump compression_hypertable "
# below tests are tracked as part of #4833
"telemetry_stats dist_commands dist_ddl dist_query dist_partial_agg continuous_aggs continuous_aggs_deprecated "
# below tests are tracked as part of #4834
"compression compression_dml modify_exclusion rowsecurity-15 "
# below tests are tracked as part of #4835
"parallel plan_hashagg query partialize_finalize dist_distinct dist_fetcher_type dist_remote_error jit-15 "
# below tests are tracked as part of #4837
"remote_txn'"}))
m["include"].append(build_without_telemetry({"pg": PG14_LATEST}))

m["include"].append(
build_debug_config(
{
"pg": 15,
"snapshot": "snapshot",
"tsdb_build_args": "-DASSERTIONS=ON -DREQUIRE_ALL_TESTS=ON -DEXPERIMENTAL=ON -DCODECOVERAGE=ON",
# below tests are tracked as part of #4838
"installcheck_args": "SKIPS='003_connections_privs 001_simple_multinode 004_multinode_rdwr_1pc data_node_bootstrap dist_hypertable-15 bgw_custom' "
# below tests are tracked as part of #4832
"IGNORES='pg_dump_unprivileged cagg_dump metadata pg_dump compression_hypertable "
# below tests are tracked as part of #4833
"telemetry_stats dist_commands dist_ddl dist_query dist_partial_agg continuous_aggs continuous_aggs_deprecated "
# below tests are tracked as part of #4834
"compression compression_dml modify_exclusion rowsecurity-15 "
# below tests are tracked as part of #4835
"parallel plan_hashagg query partialize_finalize dist_distinct dist_fetcher_type dist_remote_error jit-15 "
# below tests are tracked as part of #4837
"remote_txn'",
}
)
)

# if this is not a pull request e.g. a scheduled run or a push
# to a specific branch like prerelease_test we add additional
# entries to the matrix
if event_type != "pull_request":

# add debug test for first supported PG12 version
# most of the IGNORES are the isolation tests because the output format has changed between versions
# chunk_utils, telemetry and tablespace are skipped because of use after free bugs in postgres 12.0 which those tests hit
pg12_debug_earliest = {
"pg": PG12_EARLIEST,
"installcheck_args": "SKIPS='chunk_utils tablespace telemetry' IGNORES='cluster-12 cagg_policy debug_notice dist_gapfill_pushdown-12'",
"tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
}
m["include"].append(build_debug_config(pg12_debug_earliest))

# add debug test for first supported PG13 version
pg13_debug_earliest = {
"pg": PG13_EARLIEST,
"installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'",
"tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
}
m["include"].append(build_debug_config(pg13_debug_earliest))

# add debug test for first supported PG14 version
m["include"].append(build_debug_config({"pg": PG14_EARLIEST, "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'"}))

# add debug test for MacOS
m["include"].append(build_debug_config(macos_config({})))

# add release test for latest pg 12 and 13
m["include"].append(build_release_config({"pg":PG12_LATEST}))
m["include"].append(build_release_config({"pg":PG13_LATEST}))
m["include"].append(build_release_config({"pg":PG14_LATEST}))

# add apache only test for latest pg
m["include"].append(build_apache_config({"pg":PG12_LATEST}))
m["include"].append(build_apache_config({"pg":PG13_LATEST}))
m["include"].append(build_apache_config({"pg":PG14_LATEST}))

# to discover issues with upcoming releases we run CI against
# the stable branches of supported PG releases
m["include"].append(build_debug_config({"pg":12,"snapshot":"snapshot"}))
m["include"].append(build_debug_config({"pg":13,"snapshot":"snapshot"}))
m["include"].append(build_debug_config({"pg":14,"snapshot":"snapshot", "installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'"}))
# add debug test for first supported PG12 version
# most of the IGNORES are the isolation tests because the output format has changed between versions
# chunk_utils, telemetry and tablespace are skipped because of use after free bugs in postgres 12.0 which those tests hit
pg12_debug_earliest = {
"pg": PG12_EARLIEST,
# The early releases don't build with llvm 14.
"pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
"installcheck_args": "SKIPS='chunk_utils tablespace telemetry' IGNORES='cluster-12 cagg_policy debug_notice dist_gapfill_pushdown-12'",
"tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
}
m["include"].append(build_debug_config(pg12_debug_earliest))

# add debug test for first supported PG13 version
pg13_debug_earliest = {
"pg": PG13_EARLIEST,
# The early releases don't build with llvm 14.
"pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
"installcheck_args": "SKIPS='001_extension' IGNORES='dist_gapfill_pushdown-13'",
"tsdb_build_args": "-DCODECOVERAGE=ON -DWARNINGS_AS_ERRORS=ON -DASSERTIONS=ON -DPG_ISOLATION_REGRESS=OFF",
}
m["include"].append(build_debug_config(pg13_debug_earliest))

# add debug test for first supported PG14 version
m["include"].append(
build_debug_config(
{
"pg": PG14_EARLIEST,
# The early releases don't build with llvm 14.
"pg_extra_args": "--enable-debug --enable-cassert --without-llvm",
"installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'",
}
)
)

# add debug test for MacOS
m["include"].append(build_debug_config(macos_config({})))

# add release test for latest pg 12 and 13
m["include"].append(build_release_config({"pg": PG12_LATEST}))
m["include"].append(build_release_config({"pg": PG13_LATEST}))
m["include"].append(build_release_config({"pg": PG14_LATEST}))

# add apache only test for latest pg
m["include"].append(build_apache_config({"pg": PG12_LATEST}))
m["include"].append(build_apache_config({"pg": PG13_LATEST}))
m["include"].append(build_apache_config({"pg": PG14_LATEST}))

# to discover issues with upcoming releases we run CI against
# the stable branches of supported PG releases
m["include"].append(build_debug_config({"pg": 12, "snapshot": "snapshot"}))
m["include"].append(build_debug_config({"pg": 13, "snapshot": "snapshot"}))
m["include"].append(
build_debug_config(
{
"pg": 14,
"snapshot": "snapshot",
"installcheck_args": "IGNORES='dist_gapfill_pushdown-14 memoize'",
}
)
)

# generate command to set github action variable
with open(os.environ['GITHUB_OUTPUT'], "a") as output:
print(str.format("matrix={0}",json.dumps(m)), file=output)

with open(os.environ["GITHUB_OUTPUT"], "a") as output:
print(str.format("matrix={0}", json.dumps(m)), file=output)
13 changes: 7 additions & 6 deletions .github/workflows/linux-build-and-test.yaml
View file
Open in desktop
Expand Up @@ -38,8 +38,12 @@ jobs:
- name: Install Linux Dependencies
if: runner.os == 'Linux'
run: |
# Don't add ddebs here because the ddebs mirror is always 503 Service Unavailable.
# If needed, install them before opening the core dump.
sudo apt-get update
sudo apt-get install flex bison lcov systemd-coredump gdb libipc-run-perl libtest-most-perl ${{ matrix.extra_packages }}
# The GCC 11 gcov segfaults with the coverage info generated by clang.
sudo ln -sf $(which llvm-cov-14) $(which gcov)

- name: Install macOS Dependencies
if: runner.os == 'macOS'
Expand Down Expand Up @@ -80,12 +84,7 @@ jobs:
mkdir -p ~/$PG_SRC_DIR
tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1
cd ~/$PG_SRC_DIR
if [[ "${{ runner.os }}" == "Linux" ]]; then
./configure --prefix=$HOME/$PG_INSTALL_DIR ${{ matrix.pg_build_args }} --with-llvm LLVM_CONFIG=${{ matrix.llvm_config }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }}
else
# the current github macos image has a buggy llvm installation so we build without llvm on mac
./configure --prefix=$HOME/$PG_INSTALL_DIR ${{ matrix.pg_build_args }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }}
fi
./configure --prefix=$HOME/$PG_INSTALL_DIR --with-llvm LLVM_CONFIG=${{ matrix.llvm_config }} --with-openssl --without-readline --without-zlib --without-libxml ${{ matrix.pg_extra_args }}
make -j $MAKE_JOBS
make -j $MAKE_JOBS -C src/test/isolation
make -j $MAKE_JOBS -C contrib/postgres_fdw
Expand Down Expand Up @@ -169,6 +168,8 @@ jobs:
if: always() && steps.collectlogs.outputs.coredumps == 'true'
run: |
sudo coredumpctl gdb <<EOT
set verbose on
show debug-file-directory
printf "%s\n\n", debug_query_string
bt full
EOT
Expand Down