Skip to content

Commit 9b4d75e

Browse files
authored
chore: log upgrade failures remotely to project (#635)
* chore: log upgrade failures remotely to project table * chore: nits * chore bump version
1 parent d189bbe commit 9b4d75e

File tree

4 files changed

+105
-25
lines changed

4 files changed

+105
-25
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#! /usr/bin/env bash
2+
3+
# Common functions and variables used by pg_upgrade_initiate.sh and pg_upgrade_complete.sh
4+
5+
REPORTING_PROJECT_REF="ihmaxnjpcccasmrbkpvo"
6+
REPORTING_CREDENTIALS_FILE="/root/upgrade-reporting-credentials"
7+
8+
REPORTING_ANON_KEY=""
9+
if [ -f "$REPORTING_CREDENTIALS_FILE" ]; then
10+
REPORTING_ANON_KEY=$(cat "$REPORTING_CREDENTIALS_FILE")
11+
fi
12+
13+
function run_sql {
14+
psql -h localhost -U supabase_admin -d postgres "$@"
15+
}
16+
17+
function ship_logs {
18+
LOG_FILE=$1
19+
20+
if [ -z "$REPORTING_ANON_KEY" ]; then
21+
echo "No reporting key found. Skipping log upload."
22+
return 0
23+
fi
24+
25+
if [ ! -f "$LOG_FILE" ]; then
26+
echo "No log file found. Skipping log upload."
27+
return 0
28+
fi
29+
30+
if [ ! -s "$LOG_FILE" ]; then
31+
echo "Log file is empty. Skipping log upload."
32+
return 0
33+
fi
34+
35+
HOSTNAME=$(hostname)
36+
DERIVED_REF="${HOSTNAME##*-}"
37+
38+
printf -v BODY '{ "ref": "%s", "step": "%s", "content": %s }' "$DERIVED_REF" "completion" "$(cat "$LOG_FILE" | jq -Rs '.')"
39+
curl -sf -X POST "https://$REPORTING_PROJECT_REF.supabase.co/rest/v1/error_logs" \
40+
-H "apikey: ${REPORTING_ANON_KEY}" \
41+
-H 'Content-type: application/json' \
42+
-d "$BODY"
43+
}
44+
45+
function retry {
46+
local retries=$1
47+
shift
48+
49+
local count=0
50+
until "$@"; do
51+
exit=$?
52+
wait=$((2 ** (count + 1)))
53+
count=$((count + 1))
54+
if [ $count -lt "$retries" ]; then
55+
echo "Command $* exited with code $exit, retrying..."
56+
sleep $wait
57+
else
58+
echo "Command $* exited with code $exit, no more retries left."
59+
return $exit
60+
fi
61+
done
62+
return 0
63+
}

ansible/files/admin_api_scripts/pg_upgrade_complete.sh

100644100755
+31-19
Original file line numberDiff line numberDiff line change
@@ -7,16 +7,19 @@
77

88
set -eEuo pipefail
99

10-
run_sql() {
11-
psql -h localhost -U supabase_admin -d postgres "$@"
12-
}
10+
# shellcheck disable=SC1091
11+
source ./pg_upgrade_common.sh
12+
13+
LOG_FILE="/tmp/pg-upgrade-complete.log"
1314

14-
cleanup() {
15+
function cleanup {
1516
UPGRADE_STATUS=${1:-"failed"}
1617
EXIT_CODE=${?:-0}
1718

1819
echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status
1920

21+
ship_logs "$LOG_FILE" || true
22+
2023
exit "$EXIT_CODE"
2124
}
2225

@@ -29,41 +32,50 @@ function complete_pg_upgrade {
2932
echo "running" > /tmp/pg-upgrade-status
3033

3134
echo "1. Mounting data disk"
32-
mount -a -v
35+
retry 3 mount -a -v
3336

3437
# copying custom configurations
3538
echo "2. Copying custom configurations"
36-
cp -R /data/conf/* /etc/postgresql-custom/
37-
chown -R postgres:postgres /var/lib/postgresql/data
38-
chown -R postgres:postgres /data/pgdata
39+
retry 3 copy_configs
3940

4041
echo "3. Starting postgresql"
41-
service postgresql start
42+
retry 3 service postgresql start
4243

4344
echo "4. Running generated SQL files"
45+
retry 3 run_generated_sql
46+
47+
sleep 5
48+
49+
echo "5. Restarting postgresql"
50+
retry 3 service postgresql restart
51+
52+
echo "6. Starting vacuum analyze"
53+
retry 3 start_vacuum_analyze
54+
}
55+
56+
function copy_configs {
57+
cp -R /data/conf/* /etc/postgresql-custom/
58+
chown -R postgres:postgres /var/lib/postgresql/data
59+
chown -R postgres:postgres /data/pgdata
60+
}
61+
62+
function run_generated_sql {
4463
if [ -d /data/sql ]; then
4564
for FILE in /data/sql/*.sql; do
4665
if [ -f "$FILE" ]; then
4766
run_sql -f "$FILE"
4867
fi
4968
done
5069
fi
51-
52-
sleep 5
53-
54-
echo "5. Restarting postgresql"
55-
service postgresql restart
56-
57-
echo "6. Starting vacuum analyze"
58-
start_vacuum_analyze
5970
}
6071

6172
function start_vacuum_analyze {
73+
echo "complete" > /tmp/pg-upgrade-status
6274
su -c 'vacuumdb --all --analyze-in-stages' -s "$SHELL" postgres
6375
echo "Upgrade job completed"
64-
cleanup "complete"
6576
}
6677

6778
trap cleanup ERR
6879

69-
complete_pg_upgrade >>/var/log/pg-upgrade-complete.log 2>&1 &
80+
81+
complete_pg_upgrade >> $LOG_FILE 2>&1 &

ansible/files/admin_api_scripts/pg_upgrade_initiate.sh

+10-5
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ PG13_EXTENSIONS_TO_DISABLE=(
2727

2828
set -eEuo pipefail
2929

30+
# shellcheck disable=SC1091
31+
source ./pg_upgrade_common.sh
32+
33+
LOG_FILE="/var/log/pg-upgrade-initiate.log"
34+
3035
PGVERSION=$1
3136
IS_DRY_RUN=${2:-false}
3237
if [ "$IS_DRY_RUN" != false ]; then
@@ -35,12 +40,9 @@ fi
3540

3641
MOUNT_POINT="/data_migration"
3742

38-
run_sql() {
39-
psql -h localhost -U supabase_admin -d postgres "$@"
40-
}
41-
4243
POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql"
4344
OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;")
45+
4446
# If upgrading from older major PG versions, disable specific extensions
4547
if [[ "$OLD_PGVERSION" =~ 14* ]]; then
4648
EXTENSIONS_TO_DISABLE+=("${PG14_EXTENSIONS_TO_DISABLE[@]}")
@@ -61,6 +63,9 @@ cleanup() {
6163
if [ -d "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" ]; then
6264
echo "Copying pg_upgrade output to /var/log"
6365
cp -R "${MOUNT_POINT}/pgdata/pg_upgrade_output.d/" /var/log/ || true
66+
ship_logs "$LOG_FILE" || true
67+
tail -n +1 /var/log/pg_upgrade_output.d/*/* > /var/log/pg_upgrade_output.d/pg_upgrade.log || true
68+
ship_logs "/var/log/pg_upgrade_output.d/pg_upgrade.log" || true
6469
fi
6570

6671
if [ -L /var/lib/postgresql ]; then
@@ -279,6 +284,6 @@ echo "running" > /tmp/pg-upgrade-status
279284
if [ "$IS_DRY_RUN" = true ]; then
280285
initiate_upgrade
281286
else
282-
initiate_upgrade >> /var/log/pg-upgrade-initiate.log 2>&1 &
287+
initiate_upgrade >> "$LOG_FILE" 2>&1 &
283288
echo "Upgrade initiate job completed"
284289
fi

common.vars.pkr.hcl

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
postgres-version = "15.1.0.79"
1+
postgres-version = "15.1.0.80"

0 commit comments

Comments
 (0)