Skip to content

Commit f6fc401

Browse files
hunleydhunleyd
committed
Merge remote-tracking branch 'origin/develop' into INDATA-152
* origin/develop: chore: bump Wrappers version to 0.5.7 (#1945) chore: bump versions (#1949) Revert "docs: create an automated changlog file that shows up in release file…" (#1948) chore: remove precreated Storage objects (#1911) refactor: Move PostgreSQL auto_explain config to conf.d (#1943) docs: create an automated changlog file that shows up in release files (#1793) fix: removing dead nix code (#1946) docs: expansion to help understand anatomy and code api in user here (#1944) fix: try to use mirror fallbacks for docker images (#1942) feat: need this workflow present to test (#1941) feat: Add PostgreSQL conf.d directory (#1929) [jj-spr] initial version (#1866) chore: upgrade supabase-admin-agent (#1935) fix: garbage collect after every profile intall + remove osquery (#1938) fix(ci): limit max-jobs of nix to 8 to prevent OOM while running nix flake check (#1933) Revert "fix: disable temporary Nix checks on aarch64-linux" (#1934) fix: adaptive retry on aws (#1917) fix: disable temporary Nix checks on aarch64-linux (#1930) feat: release orioledb with rewind functionality (#1880) feat: reduciton of closure size for plv8 (#1925)
2 parents 7679ee5 + 75c8517 commit f6fc401

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+15883
-15814
lines changed

‎.github/actions/nix-install-ephemeral/action.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,4 @@ runs:
4444
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
4545
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
4646
${{ inputs.push-to-cache == 'true' && 'post-build-hook = /etc/nix/upload-to-cache.sh' || '' }}
47+
max-jobs = 4

‎.github/workflows/base-image-nightly.yml‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
name: Build Base Image Nightly
2+
3+
on:
4+
#schedule:
5+
# - cron: '0 2 * * *' # 2 AM UTC daily
6+
workflow_dispatch:
7+
inputs:
8+
branch:
9+
description: 'Branch to build from'
10+
required: false
11+
default: 'develop'
12+
type: string
13+
14+
permissions:
15+
contents: read
16+
id-token: write
17+
18+
jobs:
19+
build-base-image:
20+
runs-on: blacksmith-4vcpu-ubuntu-2404-arm
21+
timeout-minutes: 150
22+
23+
steps:
24+
- name: Checkout Repo
25+
uses: supabase/postgres/.github/actions/shared-checkout@HEAD
26+
with:
27+
ref: ${{ github.event.inputs.branch || 'develop' }}
28+
29+
- name: Configure AWS credentials
30+
uses: aws-actions/configure-aws-credentials@v4
31+
with:
32+
role-to-assume: ${{ secrets.DEV_AWS_ROLE }}
33+
aws-region: "us-east-1"
34+
output-credentials: true
35+
role-duration-seconds: 7200
36+
37+
- name: Install nix
38+
uses: cachix/install-nix-action@v27
39+
with:
40+
install_url: https://releases.nixos.org/nix/nix-2.29.1/install
41+
extra_nix_config: |
42+
substituters = https://cache.nixos.org https://nix-postgres-artifacts.s3.amazonaws.com
43+
trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
44+
45+
- name: Set execution ID and timestamp
46+
run: |
47+
echo "EXECUTION_ID=${{ github.run_id }}-base-nightly" >> $GITHUB_ENV
48+
echo "BUILD_TIMESTAMP=$(date -u +%Y%m%d-%H%M%S)" >> $GITHUB_ENV
49+
50+
- name: Build base stage 1 AMI
51+
env:
52+
AWS_MAX_ATTEMPTS: 10
53+
AWS_RETRY_MODE: adaptive
54+
run: |
55+
GIT_SHA=${{ github.sha }}
56+
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
57+
nix run github:supabase/postgres/${GIT_SHA}#packer -- build \
58+
-var "git-head-version=${GIT_SHA}" \
59+
-var "packer-execution-id=${EXECUTION_ID}" \
60+
-var-file="development-arm.vars.pkr.hcl" \
61+
-var "base-image-nightly=true" \
62+
-var "build-timestamp=${BUILD_TIMESTAMP}" \
63+
-var "region=us-east-1" \
64+
-var 'ami_regions=["us-east-1","ap-southeast-1"]' \
65+
amazon-arm64-nix.pkr.hcl
66+
67+
- name: Slack Notification on Failure
68+
if: ${{ failure() }}
69+
uses: rtCamp/action-slack-notify@v2
70+
env:
71+
SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }}
72+
SLACK_USERNAME: 'gha-failures-notifier'
73+
SLACK_COLOR: 'danger'
74+
SLACK_MESSAGE: 'Building base image nightly failed'
75+
SLACK_FOOTER: ''
76+
77+
- name: Cleanup resources after build
78+
if: ${{ always() }}
79+
run: |
80+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids
81+
82+
- name: Cleanup resources on build cancellation
83+
if: ${{ cancelled() }}
84+
run: |
85+
aws ec2 --region us-east-1 describe-instances --filters "Name=tag:packerExecutionId,Values=${EXECUTION_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --region us-east-1 --instance-ids

‎.github/workflows/nix-build.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ jobs:
2424
fail-fast: false
2525
matrix:
2626
include:
27-
- runner: blacksmith-32vcpu-ubuntu-2404
27+
- runner: blacksmith-32vcpu-ubuntu-2404
2828
arch: amd64
2929
- runner: blacksmith-32vcpu-ubuntu-2404-arm
3030
arch: arm64
@@ -66,11 +66,11 @@ jobs:
6666
sudo rm -rf /tmp/* 2>/dev/null || true
6767
echo "=== AFTER CLEANUP ==="
6868
df -h
69-
-
69+
-
7070
name: Build psql bundle
7171
run: >
72-
nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48"
73-
-- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }}
72+
nix run "github:Mic92/nix-fast-build?rev=b1dae483ab7d4139a6297e02b6de9e5d30e43d48"
73+
-- --skip-cached --no-nom ${{ matrix.runner == 'macos-latest-xlarge' && '--max-jobs 1' || '' }} --copy-to "s3://nix-postgres-artifacts?secret-key=/etc/nix/nix-secret-key"
7474
--flake ".#checks.$(nix eval --raw --impure --expr 'builtins.currentSystem')"
7575
env:
7676
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}

‎.github/workflows/testinfra-ami-build.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,12 +109,18 @@ jobs:
109109
echo "" >> common-nix.vars.pkr.hcl
110110
111111
- name: Build AMI stage 1
112+
env:
113+
AWS_MAX_ATTEMPTS: 10
114+
AWS_RETRY_MODE: adaptive
112115
run: |
113116
GIT_SHA=${{github.sha}}
114117
nix run github:supabase/postgres/${GIT_SHA}#packer -- init amazon-arm64-nix.pkr.hcl
115118
nix run github:supabase/postgres/${GIT_SHA}#packer -- build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${EXECUTION_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=" -var "postgres-version=${{ steps.random.outputs.random_string }}" -var "region=ap-southeast-1" -var 'ami_regions=["ap-southeast-1"]' -var "force-deregister=true" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl
116119
117120
- name: Build AMI stage 2
121+
env:
122+
AWS_MAX_ATTEMPTS: 10
123+
AWS_RETRY_MODE: adaptive
118124
run: |
119125
GIT_SHA=${{github.sha}}
120126
nix run github:supabase/postgres/${GIT_SHA}#packer -- init stage2-nix-psql.pkr.hcl

‎Dockerfile-15‎

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ ARG pg_repack_release=1.4.8
3030
ARG vault_release=0.2.8
3131
ARG groonga_release=12.0.8
3232
ARG pgroonga_release=2.4.0
33-
ARG wrappers_release=0.5.6
33+
ARG wrappers_release=0.5.7
3434
ARG hypopg_release=1.3.1
3535
ARG pgvector_release=0.4.0
3636
ARG pg_tle_release=1.3.2
@@ -40,7 +40,56 @@ ARG wal_g_release=2.0.1
4040

4141
FROM ubuntu:noble as base
4242

43-
RUN apt update -y && apt install -y \
43+
# Create reusable apt mirror fallback function
44+
RUN echo '#!/bin/bash\n\
45+
apt_update_with_fallback() {\n\
46+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
47+
local max_attempts=2\n\
48+
local attempt=1\n\
49+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
50+
\n\
51+
for mirror in $mirrors; do\n\
52+
echo "========================================="\n\
53+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
54+
echo "Attempt ${attempt} of ${max_attempts}"\n\
55+
echo "========================================="\n\
56+
\n\
57+
if [ -f "${sources_file}" ]; then\n\
58+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
59+
fi\n\
60+
\n\
61+
if timeout 300 apt-get update 2>&1; then\n\
62+
echo "========================================="\n\
63+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
64+
echo "========================================="\n\
65+
return 0\n\
66+
else\n\
67+
local exit_code=$?\n\
68+
echo "========================================="\n\
69+
echo "✗ Failed to update using mirror: ${mirror}"\n\
70+
echo "Exit code: ${exit_code}"\n\
71+
echo "========================================="\n\
72+
\n\
73+
apt-get clean\n\
74+
rm -rf /var/lib/apt/lists/*\n\
75+
\n\
76+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
77+
local sleep_time=$((attempt * 5))\n\
78+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
79+
sleep ${sleep_time}\n\
80+
fi\n\
81+
fi\n\
82+
\n\
83+
attempt=$((attempt + 1))\n\
84+
done\n\
85+
\n\
86+
echo "========================================="\n\
87+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
88+
echo "========================================="\n\
89+
return 1\n\
90+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
91+
92+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4493
curl \
4594
gnupg \
4695
lsb-release \
@@ -96,13 +145,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
96145
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
97146

98147

99-
RUN apt-get update && \
148+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
100149
apt-get install -y --no-install-recommends tzdata
101150

102151
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
103152
dpkg-reconfigure --frontend noninteractive tzdata
104153

105-
RUN apt-get update && \
154+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
106155
apt-get install -y --no-install-recommends \
107156
build-essential \
108157
checkinstall \
@@ -143,7 +192,7 @@ WORKDIR /
143192
FROM base as gosu
144193
ARG TARGETARCH
145194
# Install dependencies
146-
RUN apt-get update && apt-get install -y --no-install-recommends \
195+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
147196
gnupg \
148197
ca-certificates \
149198
&& rm -rf /var/lib/apt/lists/*
@@ -174,6 +223,8 @@ COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j
174223
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
175224
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
176225
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
226+
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
227+
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
177228
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
178229
COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
179230
COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
@@ -185,7 +236,6 @@ RUN \
185236
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
186237
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
187238
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
188-
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
189239
usermod -aG postgres wal-g && \
190240
mkdir -p /etc/postgresql-custom/conf.d && \
191241
chown -R postgres:postgres /etc/postgresql-custom
@@ -212,7 +262,7 @@ EXPOSE 5432
212262
ENV POSTGRES_HOST=/var/run/postgresql
213263
ENV POSTGRES_USER=supabase_admin
214264
ENV POSTGRES_DB=postgres
215-
RUN apt-get update && apt-get install -y --no-install-recommends \
265+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
216266
locales \
217267
&& rm -rf /var/lib/apt/lists/* && \
218268
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

‎Dockerfile-17‎

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ ARG pg_repack_release=1.4.8
3131
ARG vault_release=0.2.8
3232
ARG groonga_release=12.0.8
3333
ARG pgroonga_release=2.4.0
34-
ARG wrappers_release=0.5.6
34+
ARG wrappers_release=0.5.7
3535
ARG hypopg_release=1.3.1
3636
ARG pgvector_release=0.4.0
3737
ARG pg_tle_release=1.3.2
@@ -41,7 +41,56 @@ ARG wal_g_release=3.0.5
4141

4242
FROM ubuntu:noble as base
4343

44-
RUN apt update -y && apt install -y \
44+
# Create reusable apt mirror fallback function
45+
RUN echo '#!/bin/bash\n\
46+
apt_update_with_fallback() {\n\
47+
local sources_file="/etc/apt/sources.list.d/ubuntu.sources"\n\
48+
local max_attempts=2\n\
49+
local attempt=1\n\
50+
local mirrors="archive.ubuntu.com us.archive.ubuntu.com"\n\
51+
\n\
52+
for mirror in $mirrors; do\n\
53+
echo "========================================="\n\
54+
echo "Attempting apt-get update with mirror: ${mirror}"\n\
55+
echo "Attempt ${attempt} of ${max_attempts}"\n\
56+
echo "========================================="\n\
57+
\n\
58+
if [ -f "${sources_file}" ]; then\n\
59+
sed -i "s|http://[^/]*/ubuntu/|http://${mirror}/ubuntu/|g" "${sources_file}"\n\
60+
fi\n\
61+
\n\
62+
if timeout 300 apt-get update 2>&1; then\n\
63+
echo "========================================="\n\
64+
echo "✓ Successfully updated apt cache using mirror: ${mirror}"\n\
65+
echo "========================================="\n\
66+
return 0\n\
67+
else\n\
68+
local exit_code=$?\n\
69+
echo "========================================="\n\
70+
echo "✗ Failed to update using mirror: ${mirror}"\n\
71+
echo "Exit code: ${exit_code}"\n\
72+
echo "========================================="\n\
73+
\n\
74+
apt-get clean\n\
75+
rm -rf /var/lib/apt/lists/*\n\
76+
\n\
77+
if [ ${attempt} -lt ${max_attempts} ]; then\n\
78+
local sleep_time=$((attempt * 5))\n\
79+
echo "Waiting ${sleep_time} seconds before trying next mirror..."\n\
80+
sleep ${sleep_time}\n\
81+
fi\n\
82+
fi\n\
83+
\n\
84+
attempt=$((attempt + 1))\n\
85+
done\n\
86+
\n\
87+
echo "========================================="\n\
88+
echo "ERROR: All mirror tiers failed after ${max_attempts} attempts"\n\
89+
echo "========================================="\n\
90+
return 1\n\
91+
}' > /usr/local/bin/apt-update-fallback.sh && chmod +x /usr/local/bin/apt-update-fallback.sh
92+
93+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt install -y \
4594
curl \
4695
gnupg \
4796
lsb-release \
@@ -100,13 +149,13 @@ RUN chown -R postgres:postgres /usr/lib/postgresql
100149
RUN ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets
101150

102151

103-
RUN apt-get update && \
152+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
104153
apt-get install -y --no-install-recommends tzdata
105154

106155
RUN ln -fs /usr/share/zoneinfo/Etc/UTC /etc/localtime && \
107156
dpkg-reconfigure --frontend noninteractive tzdata
108157

109-
RUN apt-get update && \
158+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && \
110159
apt-get install -y --no-install-recommends \
111160
build-essential \
112161
checkinstall \
@@ -148,7 +197,7 @@ WORKDIR /
148197
FROM base as gosu
149198
ARG TARGETARCH
150199
# Install dependencies
151-
RUN apt-get update && apt-get install -y --no-install-recommends \
200+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
152201
gnupg \
153202
ca-certificates \
154203
&& rm -rf /var/lib/apt/lists/*
@@ -179,6 +228,8 @@ COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j
179228
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf
180229
COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf
181230
COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/
231+
COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf
232+
COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
182233
COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
183234
COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
184235
COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
@@ -190,7 +241,6 @@ RUN \
190241
#echo "pljava.libjvm_location = '/usr/lib/jvm/java-11-openjdk-${TARGETARCH}/lib/server/libjvm.so'" >> /etc/postgresql/postgresql.conf && \
191242
echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
192243
echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
193-
echo 'auto_explain.log_min_duration = 10s' >> /etc/postgresql/postgresql.conf && \
194244
usermod -aG postgres wal-g && \
195245
mkdir -p /etc/postgresql-custom/conf.d && \
196246
chown -R postgres:postgres /etc/postgresql-custom
@@ -226,7 +276,7 @@ ENV POSTGRES_HOST=/var/run/postgresql
226276
ENV POSTGRES_USER=supabase_admin
227277
ENV POSTGRES_DB=postgres
228278
ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8"
229-
RUN apt-get update && apt-get install -y --no-install-recommends \
279+
RUN bash -c 'source /usr/local/bin/apt-update-fallback.sh && apt_update_with_fallback' && apt-get install -y --no-install-recommends \
230280
locales \
231281
&& rm -rf /var/lib/apt/lists/* && \
232282
localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 \

0 commit comments

Comments
 (0)