diff --git a/circleci/images/Makefile b/circleci/images/Makefile index 6b91a55..b35c2cd 100644 --- a/circleci/images/Makefile +++ b/circleci/images/Makefile @@ -2,6 +2,11 @@ DOCKER_REPO ?= ghcr.io/citusdata SHA_SUFFIX := $(shell git rev-parse --short HEAD) +# Upstream postgres git ref used to source-build PG19 (which is unreleased +# and therefore not available via apt.postgresql.org). Override at build +# time, e.g. `make PG19_SOURCE_REF=master push-extbuilder-19devel`. +PG19_SOURCE_REF ?= 43649b6a53e9b3964e921cc962f5c977bce688fd + # auto generated variables ifdef RELEASE TAG_SUFFIX := -v${SHA_SUFFIX} @@ -16,8 +21,11 @@ PG_UPGRADE_TESTER_VERSION=$(shell echo ${PG_VERSIONS}|tr ' ' '-'|sed 's/~//g') STYLE_CHECKER_TOOLS_VERSION=0.8.33 -# Upgrade tests for the PG major versions from PG_VERSIONS file -CITUS_UPGRADE_PG_VERSIONS=$(shell head PG_VERSIONS|cut -c 6-|tr '\n' ' ') +# Upgrade tests for the PG major versions from PG_VERSIONS file. +# PG19 is excluded because no Citus release supports it yet; filter it +# out by name so reordering PG_VERSIONS does not silently change which +# major is dropped. +CITUS_UPGRADE_PG_VERSIONS=$(shell grep -v '^PG19=' PG_VERSIONS|cut -c 6-|tr '\n' ' ') # 12.1.10 is the latest release of Citus 12 when the test is expanded to cover Citus 12 CITUS_UPGRADE_VERSIONS_16=v12.1.12 # Latest minor version of Citus 13 @@ -43,6 +51,7 @@ build-extbuilder-$1: -f extbuilder/Dockerfile \ --build-arg=PG_VERSION=$1 \ --build-arg=PG_MAJOR=$2 \ + --build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \ --tag=${DOCKER_REPO}/extbuilder:$3${TAG_SUFFIX} build-all:: build-extbuilder-$1 @@ -62,6 +71,7 @@ build-exttester-$1: --build-arg=PG_VERSION=$1 \ --build-arg=PG_MAJOR=$2 \ --build-arg=PG_VERSION_CLEAN=$3 \ + --build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \ --build-arg=CITUS_VERSION="$(call get_last_citus_version,$2)" \ --tag=${DOCKER_REPO}/exttester:$3${TAG_SUFFIX} @@ -82,6 +92,7 @@ build-failtester-$1: -f failtester/Dockerfile \ --build-arg=PG_VERSION=$1 \ --build-arg=PG_MAJOR=$2 \ + --build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \ --tag=${DOCKER_REPO}/failtester:$3${TAG_SUFFIX} build-all:: build-failtester-$1 @@ -110,6 +121,7 @@ build-citusupgradetester-$1: -f citusupgradetester/Dockerfile \ --build-arg=PG_VERSION=$1 \ --build-arg=PG_MAJOR=$2 \ + --build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \ --build-arg=CITUS_VERSIONS="$(call get_citus_versions,$2)" \ --tag=${DOCKER_REPO}/citusupgradetester:$3${TAG_SUFFIX} @@ -131,6 +143,7 @@ build-pgupgradetester: pgupgradetester/ \ -f pgupgradetester/Dockerfile \ --build-arg=PG_VERSIONS="${PG_VERSIONS}" \ + --build-arg=PG19_SOURCE_REF=${PG19_SOURCE_REF} \ --tag=${DOCKER_REPO}/pgupgradetester:${PG_UPGRADE_TESTER_VERSION}${TAG_SUFFIX} build-all:: build-pgupgradetester diff --git a/circleci/images/PG_VERSIONS b/circleci/images/PG_VERSIONS index 98c38dc..7054c61 100644 --- a/circleci/images/PG_VERSIONS +++ b/circleci/images/PG_VERSIONS @@ -1,3 +1,4 @@ -PG16=16.13 -PG17=17.9 -PG18=18.3 +PG16=16.14 +PG17=17.10 +PG18=18.4 +PG19=19devel diff --git a/circleci/images/citusupgradetester/Dockerfile b/circleci/images/citusupgradetester/Dockerfile index b434779..9cd7624 100644 --- a/circleci/images/citusupgradetester/Dockerfile +++ b/circleci/images/citusupgradetester/Dockerfile @@ -45,31 +45,85 @@ RUN locale-gen ARG PG_VERSION ARG PG_MAJOR +ARG PG19_SOURCE_REF ENV PG_VERSION=$PG_VERSION ENV PG_MAJOR=$PG_MAJOR +ENV PG19_SOURCE_REF=$PG19_SOURCE_REF RUN <<'EOF' # install postgres ecosystem for pg version: $PG_VERSION set -eux -# install key and repositories -curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - -echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list -echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list - -apt-get update - -# infer the pgdgversion of postgres based on the $PG_VERSION -pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) - -apt-get install -y --no-install-recommends --allow-downgrades \ - libpq-dev=${pgdg_version} \ - libpq5=${pgdg_version} \ - postgresql-${PG_MAJOR}=${pgdg_version} \ - postgresql-client-${PG_MAJOR}=${pgdg_version} \ - postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ - postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \ +if [ "${PG_MAJOR}" = "19" ]; then + # PG19 is unreleased and not on apt.postgresql.org. Build from a pinned + # upstream source ref so this image can still be produced. Note that no + # Citus release currently supports PG19, so the Makefile excludes PG19 + # from CITUS_UPGRADE_PG_VERSIONS and this branch should not be hit by + # CI -- it exists so manual builds don't fail noisily. + apt-get update + apt-get install -y --no-install-recommends \ + bison \ + flex \ + libxml2-dev \ + pkg-config \ + zlib1g-dev + + src_dir=/tmp/pg19-src + rm -rf "${src_dir}" + mkdir -p "${src_dir}" + curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \ + | tar -xzf - -C "${src_dir}" --strip-components=1 + cd "${src_dir}" + ./configure \ + --prefix=/usr/lib/postgresql/${PG_MAJOR} \ + --datarootdir=/usr/share/postgresql/${PG_MAJOR} \ + --includedir=/usr/include/postgresql/${PG_MAJOR} \ + --sysconfdir=/etc/postgresql-common \ + --with-openssl \ + --with-icu \ + --with-libxml \ + --with-libxslt \ + --with-lz4 \ + --with-zstd \ + --enable-rpath \ + --with-system-tzdata=/usr/share/zoneinfo + make -sj"$(nproc)" world-bin + make install-world-bin + cd / + rm -rf "${src_dir}" + + # apt-installed postgres packages normally create this user/group; do + # it manually for the source-build path so the chown in the next RUN + # step still works. + if ! getent group postgres >/dev/null; then + groupadd --system postgres + fi + if ! id postgres >/dev/null 2>&1; then + useradd --system --gid postgres --home-dir /var/lib/postgresql --shell /bin/bash postgres + fi + mkdir -p /var/lib/postgresql + chown -R postgres:postgres /var/lib/postgresql +else + # install key and repositories + curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - + echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list + echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list + + apt-get update + + # infer the pgdgversion of postgres based on the $PG_VERSION + pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) + + apt-get install -y --no-install-recommends --allow-downgrades \ + libpq-dev=${pgdg_version} \ + libpq5=${pgdg_version} \ + postgresql-${PG_MAJOR}=${pgdg_version} \ + postgresql-client-${PG_MAJOR}=${pgdg_version} \ + postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ + postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \ + +fi # clear apt cache rm -rf /var/lib/apt/lists/* diff --git a/circleci/images/extbuilder/Dockerfile b/circleci/images/extbuilder/Dockerfile index d703a28..f6c3a85 100644 --- a/circleci/images/extbuilder/Dockerfile +++ b/circleci/images/extbuilder/Dockerfile @@ -39,34 +39,76 @@ EOF ARG PG_VERSION ARG PG_MAJOR +ARG PG19_SOURCE_REF ENV PG_VERSION=$PG_VERSION ENV PG_MAJOR=$PG_MAJOR +ENV PG19_SOURCE_REF=$PG19_SOURCE_REF RUN <<'EOF' # install postgres ecosystem for pg version: $PG_VERSION set -eux -# install key and repositories -curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - -echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list -echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list +if [ "${PG_MAJOR}" = "19" ]; then + # PG19 is unreleased and not on apt.postgresql.org. Build from a pinned + # upstream source ref so this image can still be produced. + apt-get update + apt-get install -y --no-install-recommends \ + bison \ + flex \ + libxml2-dev \ + pkg-config \ + zlib1g-dev -apt-get update + src_dir=/tmp/pg19-src + rm -rf "${src_dir}" + mkdir -p "${src_dir}" + curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \ + | tar -xzf - -C "${src_dir}" --strip-components=1 + cd "${src_dir}" + ./configure \ + --prefix=/usr/lib/postgresql/${PG_MAJOR} \ + --datarootdir=/usr/share/postgresql/${PG_MAJOR} \ + --includedir=/usr/include/postgresql/${PG_MAJOR} \ + --sysconfdir=/etc/postgresql-common \ + --with-openssl \ + --with-icu \ + --with-libxml \ + --with-libxslt \ + --with-lz4 \ + --with-zstd \ + --enable-rpath \ + --with-system-tzdata=/usr/share/zoneinfo + make -sj"$(nproc)" world-bin + make install-world-bin + cd / + rm -rf "${src_dir}" +else + # install key and repositories + curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - + echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list + echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list + + apt-get update -# infer the pgdgversion of postgres based on the $PG_VERSION -pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) + # infer the pgdgversion of postgres based on the $PG_VERSION + pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) -apt-get install -y --no-install-recommends --allow-downgrades \ - libpq-dev=${pgdg_version} \ - libpq5=${pgdg_version} \ - postgresql-${PG_MAJOR}=${pgdg_version} \ - postgresql-client-${PG_MAJOR}=${pgdg_version} \ - postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ - postgresql-server-dev-${PG_MAJOR}=${pgdg_version} + apt-get install -y --no-install-recommends --allow-downgrades \ + libpq-dev=${pgdg_version} \ + libpq5=${pgdg_version} \ + postgresql-${PG_MAJOR}=${pgdg_version} \ + postgresql-client-${PG_MAJOR}=${pgdg_version} \ + postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ + postgresql-server-dev-${PG_MAJOR}=${pgdg_version} +fi # clear apt cache rm -rf /var/lib/apt/lists/* EOF +# make pg_config (installed by postgresql-server-dev-${PG_MAJOR} from apt or +# by the PG19 source-build path) discoverable to extension builds +ENV PATH=/usr/lib/postgresql/$PG_MAJOR/bin/:$PATH + USER circleci WORKDIR /home/circleci diff --git a/circleci/images/exttester/Dockerfile b/circleci/images/exttester/Dockerfile index 4af0f75..843b64e 100644 --- a/circleci/images/exttester/Dockerfile +++ b/circleci/images/exttester/Dockerfile @@ -42,11 +42,21 @@ EOF ARG PG_VERSION ARG PG_VERSION_CLEAN ARG PG_MAJOR +ARG PG19_SOURCE_REF WORKDIR /build/ -RUN curl -fLO "http://ftp.postgresql.org/pub/source/v${PG_VERSION_CLEAN}/postgresql-${PG_VERSION_CLEAN}.tar.bz2" - -RUN tar jxf "postgresql-${PG_VERSION_CLEAN}.tar.bz2" +RUN <<'EOF' +set -eux +if [ "${PG_MAJOR}" = "19" ]; then + # PG19 is unreleased -- fetch the source from a pinned upstream git ref + mkdir -p "postgresql-${PG_VERSION_CLEAN}" + curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \ + | tar -xzf - -C "postgresql-${PG_VERSION_CLEAN}" --strip-components=1 +else + curl -fLO "http://ftp.postgresql.org/pub/source/v${PG_VERSION_CLEAN}/postgresql-${PG_VERSION_CLEAN}.tar.bz2" + tar jxf "postgresql-${PG_VERSION_CLEAN}.tar.bz2" +fi +EOF # apply optional patches that might be required for a successful testsuite WORKDIR /build/postgresql-${PG_VERSION_CLEAN}/ @@ -142,36 +152,91 @@ RUN locale-gen ARG PG_VERSION ARG PG_MAJOR +ARG PG19_SOURCE_REF ARG CITUS_VERSION ENV PG_VERSION=$PG_VERSION ENV PG_MAJOR=$PG_MAJOR +ENV PG19_SOURCE_REF=$PG19_SOURCE_REF ENV CITUS_VERSION=$CITUS_VERSION RUN <<'EOF' # install postgres ecosystem for pg version: $PG_VERSION set -eux -# install key and repositories -curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - -echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list -echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list - -apt-get update - -# infer the pgdgversion of postgres based on the $PG_VERSION -pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) - -apt-get install -y --no-install-recommends --allow-downgrades \ - libdbi-perl \ - libpq-dev=${pgdg_version} \ - libpq5=${pgdg_version} \ - postgresql-${PG_MAJOR}=${pgdg_version} \ - postgresql-client-${PG_MAJOR}=${pgdg_version} \ - postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ - postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \ - postgresql-${PG_MAJOR}-wal2json - -apt-get install -y --no-install-recommends -t bookworm libdbd-pg-perl +if [ "${PG_MAJOR}" = "19" ]; then + # PG19 is unreleased and not on apt.postgresql.org. Build from a pinned + # upstream source ref so this image can still be produced. + apt-get update + apt-get install -y --no-install-recommends \ + bison \ + flex \ + libdbi-perl \ + libxml2-dev \ + pkg-config \ + zlib1g-dev + apt-get install -y --no-install-recommends -t bookworm libdbd-pg-perl + + src_dir=/tmp/pg19-src + rm -rf "${src_dir}" + mkdir -p "${src_dir}" + curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \ + | tar -xzf - -C "${src_dir}" --strip-components=1 + cd "${src_dir}" + ./configure \ + --prefix=/usr/lib/postgresql/${PG_MAJOR} \ + --datarootdir=/usr/share/postgresql/${PG_MAJOR} \ + --includedir=/usr/include/postgresql/${PG_MAJOR} \ + --sysconfdir=/etc/postgresql-common \ + --with-openssl \ + --with-icu \ + --with-libxml \ + --with-libxslt \ + --with-lz4 \ + --with-zstd \ + --enable-rpath \ + --with-system-tzdata=/usr/share/zoneinfo + make -sj"$(nproc)" world-bin + make install-world-bin + cd / + rm -rf "${src_dir}" + + # apt-installed postgres packages normally create this user/group; do + # it manually for the source-build path so subsequent RUN steps that + # chown to postgres work. + if ! getent group postgres >/dev/null; then + groupadd --system postgres + fi + if ! id postgres >/dev/null 2>&1; then + useradd --system --gid postgres --home-dir /var/lib/postgresql --shell /bin/bash postgres + fi + mkdir -p /var/lib/postgresql + chown -R postgres:postgres /var/lib/postgresql + + # wal2json is not yet packaged against PG19, so we skip it for this + # branch. Existing tests that depend on it should pin to PG <= 18. +else + # install key and repositories + curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - + echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list + echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list + + apt-get update + + # infer the pgdgversion of postgres based on the $PG_VERSION + pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) + + apt-get install -y --no-install-recommends --allow-downgrades \ + libdbi-perl \ + libpq-dev=${pgdg_version} \ + libpq5=${pgdg_version} \ + postgresql-${PG_MAJOR}=${pgdg_version} \ + postgresql-client-${PG_MAJOR}=${pgdg_version} \ + postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ + postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \ + postgresql-${PG_MAJOR}-wal2json + + apt-get install -y --no-install-recommends -t bookworm libdbd-pg-perl +fi # clear apt cache rm -rf /var/lib/apt/lists/* diff --git a/circleci/images/failtester/Dockerfile b/circleci/images/failtester/Dockerfile index ac68c87..74b008a 100644 --- a/circleci/images/failtester/Dockerfile +++ b/circleci/images/failtester/Dockerfile @@ -46,29 +46,80 @@ RUN locale-gen ARG PG_VERSION ARG PG_MAJOR +ARG PG19_SOURCE_REF ENV PG_VERSION=$PG_VERSION ENV PG_MAJOR=$PG_MAJOR +ENV PG19_SOURCE_REF=$PG19_SOURCE_REF RUN <<'EOF' # install postgres ecosystem for pg version: $PG_VERSION -# install key and repositories -curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - -echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list -echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list +if [ "${PG_MAJOR}" = "19" ]; then + # PG19 is unreleased and not on apt.postgresql.org. Build from a pinned + # upstream source ref so this image can still be produced. + apt-get update + apt-get install -y --no-install-recommends \ + bison \ + flex \ + libxml2-dev \ + pkg-config \ + zlib1g-dev + + src_dir=/tmp/pg19-src + rm -rf "${src_dir}" + mkdir -p "${src_dir}" + curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \ + | tar -xzf - -C "${src_dir}" --strip-components=1 + cd "${src_dir}" + ./configure \ + --prefix=/usr/lib/postgresql/${PG_MAJOR} \ + --datarootdir=/usr/share/postgresql/${PG_MAJOR} \ + --includedir=/usr/include/postgresql/${PG_MAJOR} \ + --sysconfdir=/etc/postgresql-common \ + --with-openssl \ + --with-icu \ + --with-libxml \ + --with-libxslt \ + --with-lz4 \ + --with-zstd \ + --enable-rpath \ + --with-system-tzdata=/usr/share/zoneinfo + make -sj"$(nproc)" world-bin + make install-world-bin + cd / + rm -rf "${src_dir}" + + # apt-installed postgres packages normally create this user/group; do + # it manually for the source-build path so the chown in the next RUN + # step still works. + if ! getent group postgres >/dev/null; then + groupadd --system postgres + fi + if ! id postgres >/dev/null 2>&1; then + useradd --system --gid postgres --home-dir /var/lib/postgresql --shell /bin/bash postgres + fi + mkdir -p /var/lib/postgresql + chown -R postgres:postgres /var/lib/postgresql +else + # install key and repositories + curl -sf https://www.postgresql.org/media/keys/ACCC4CF8.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 apt-key add - + echo "deb https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" >> /etc/apt/sources.list.d/postgresql.list + echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive main" >> /etc/apt/sources.list.d/postgresql.list + + apt-get update + + # infer the pgdgversion of postgres based on the $PG_VERSION + pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) + + apt-get install -y --no-install-recommends --allow-downgrades \ + libpq-dev=${pgdg_version} \ + libpq5=${pgdg_version} \ + postgresql-${PG_MAJOR}=${pgdg_version} \ + postgresql-client-${PG_MAJOR}=${pgdg_version} \ + postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ + postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \ -apt-get update - -# infer the pgdgversion of postgres based on the $PG_VERSION -pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) - -apt-get install -y --no-install-recommends --allow-downgrades \ - libpq-dev=${pgdg_version} \ - libpq5=${pgdg_version} \ - postgresql-${PG_MAJOR}=${pgdg_version} \ - postgresql-client-${PG_MAJOR}=${pgdg_version} \ - postgresql-${PG_MAJOR}-dbgsym=${pgdg_version} \ - postgresql-server-dev-${PG_MAJOR}=${pgdg_version} \ +fi # clear apt cache rm -rf /var/lib/apt/lists/* diff --git a/circleci/images/pgupgradetester/Dockerfile b/circleci/images/pgupgradetester/Dockerfile index 9e0237d..53414bc 100644 --- a/circleci/images/pgupgradetester/Dockerfile +++ b/circleci/images/pgupgradetester/Dockerfile @@ -40,7 +40,9 @@ rm -rf /var/lib/apt/lists/* EOF ARG PG_VERSIONS +ARG PG19_SOURCE_REF ENV PG_VERSIONS=$PG_VERSIONS +ENV PG19_SOURCE_REF=$PG19_SOURCE_REF # Hacky way to fetch all the pgdg versions so we can pin them # you will find the sequence '"'"' a couple of times. This is to @@ -61,8 +63,15 @@ echo "deb https://apt-archive.postgresql.org/pub/repos/apt bookworm-pgdg-archive apt-get update pkgs=(); +source_build_majors=(); for PG_VERSION in $PG_VERSIONS; do PG_MAJOR=$(echo ${PG_VERSION} | awk -F'[^0-9]*' '/[0-9]/ { print $1 }') + if [ "${PG_MAJOR}" = "19" ]; then + # PG19 is unreleased; fall through to a source build below instead + # of asking apt for it. + source_build_majors+=("${PG_MAJOR}"); + continue; + fi pgdg_version=$(apt list -a postgresql-server-dev-${PG_MAJOR} 2>/dev/null | grep "${PG_VERSION}" | awk '{print $2}' | head -n1 ) pkgs+=("postgresql-${PG_MAJOR}=${pgdg_version}") pkgs+=("postgresql-client-${PG_MAJOR}=${pgdg_version}") @@ -71,14 +80,68 @@ for PG_VERSION in $PG_VERSIONS; do last_pgdg_version=$pgdg_version; done; -pkgs+=("libpq-dev=${last_pgdg_version}"); -pkgs+=("libpq5=${last_pgdg_version}"); - -echo ${pkgs[@]} - -apt-get install -y --no-install-recommends --allow-downgrades \ - postgresql-common \ - ${pkgs[@]} \ +if [ ${#pkgs[@]} -gt 0 ]; then + pkgs+=("libpq-dev=${last_pgdg_version}"); + pkgs+=("libpq5=${last_pgdg_version}"); + + echo ${pkgs[@]} + + apt-get install -y --no-install-recommends --allow-downgrades \ + postgresql-common \ + ${pkgs[@]} \ + +fi + +# Source-build any majors that aren't on apt.postgresql.org (currently PG19). +if [ ${#source_build_majors[@]} -gt 0 ]; then + apt-get install -y --no-install-recommends \ + bison \ + flex \ + libicu-dev \ + liblz4-dev \ + libreadline-dev \ + libssl-dev \ + libxml2-dev \ + libxslt-dev \ + libzstd-dev \ + pkg-config \ + zlib1g-dev + + if ! getent group postgres >/dev/null; then + groupadd --system postgres + fi + if ! id postgres >/dev/null 2>&1; then + useradd --system --gid postgres --home-dir /var/lib/postgresql --shell /bin/bash postgres + fi + mkdir -p /var/lib/postgresql + chown -R postgres:postgres /var/lib/postgresql + + for PG_MAJOR in "${source_build_majors[@]}"; do + src_dir=/tmp/pg${PG_MAJOR}-src + rm -rf "${src_dir}" + mkdir -p "${src_dir}" + curl -fsL "https://codeload.github.com/postgres/postgres/tar.gz/${PG19_SOURCE_REF}" \ + | tar -xzf - -C "${src_dir}" --strip-components=1 + pushd "${src_dir}" + ./configure \ + --prefix=/usr/lib/postgresql/${PG_MAJOR} \ + --datarootdir=/usr/share/postgresql/${PG_MAJOR} \ + --includedir=/usr/include/postgresql/${PG_MAJOR} \ + --sysconfdir=/etc/postgresql-common \ + --with-openssl \ + --with-icu \ + --with-libxml \ + --with-libxslt \ + --with-lz4 \ + --with-zstd \ + --enable-rpath \ + --with-system-tzdata=/usr/share/zoneinfo + make -sj"$(nproc)" world-bin + make install-world-bin + popd + rm -rf "${src_dir}" + done +fi # clear apt cache rm -rf /var/lib/apt/lists/*