Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion scripts/bats/20_check.bats
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ load 'test/libs/seeds'
load 'test/libs/sql'

setup_file() {
reset_database
init_database
}

teardown_file() {
drop_database
}

setup() {
Expand Down Expand Up @@ -49,6 +53,7 @@ EOF

assert_success
assert_output --partial "All partitions are correctly configured"
rm "$CONFIGURATION_FILE"
}

@test "Test check return an error when retention partitions are missing" {
Expand Down Expand Up @@ -79,6 +84,7 @@ EOF

assert_failure
assert_output --partial "Found missing tables"
rm "$CONFIGURATION_FILE"
}

@test "Test check return an error when preProvisioned partitions are missing" {
Expand Down Expand Up @@ -109,6 +115,7 @@ EOF

assert_failure
assert_output --partial "Found missing tables"
rm "$CONFIGURATION_FILE"
}

@test "Test check succeeding with an UUID partition key" {
Expand Down
38 changes: 29 additions & 9 deletions scripts/bats/30_provisioning.bats
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ load 'test/libs/sql'
load 'test/libs/time'

setup_file() {
reset_database
init_database
}

teardown_file() {
drop_database
}

setup() {
Expand Down Expand Up @@ -45,6 +49,8 @@ EOF
assert_table_exists public $(generate_daily_partition_name ${TABLE} -1) # retention partition
assert_table_exists public $(generate_daily_partition_name ${TABLE} 0) # current partition
assert_table_exists public $(generate_daily_partition_name ${TABLE} 1) # preProvisioned partition

rm "$CONFIGURATION_FILE"
}

@test "Test that preProvisioned and retention partitions can be increased" {
Expand Down Expand Up @@ -90,6 +96,8 @@ EOF
assert_output --partial "All partitions are correctly provisioned"
assert_table_exists public $(generate_daily_partition_name ${TABLE} -${NEW_RETENTION}) # New retention partition
assert_table_exists public $(generate_daily_partition_name ${TABLE} ${NEW_PREPROVISIONED}) # New preProvisioned partition

rm "$CONFIGURATION_FILE"
}

@test "Test monthly partitions" {
Expand Down Expand Up @@ -127,6 +135,8 @@ EOF
assert_table_exists public ${EXPECTED_LAST_TABLE}
assert_table_exists public ${EXPECTED_CURRENT_TABLE}
assert_table_exists public ${EXPECTED_NEXT_TABLE}

rm "$CONFIGURATION_FILE"
}

@test "Test quarterly partitions" {
Expand Down Expand Up @@ -157,12 +167,13 @@ EOF

run "$PPM_PROG" run provisioning -c ${CONFIGURATION_FILE}


assert_success
assert_output --partial "All partitions are correctly provisioned"
assert_table_exists public ${EXPECTED_LAST_TABLE}
assert_table_exists public ${EXPECTED_CURRENT_TABLE}
assert_table_exists public ${EXPECTED_NEXT_TABLE}

rm "$CONFIGURATION_FILE"
}

@test "Test yearly partitions" {
Expand Down Expand Up @@ -198,6 +209,8 @@ EOF
assert_table_exists public ${EXPECTED_LAST_TABLE}
assert_table_exists public ${EXPECTED_CURRENT_TABLE}
assert_table_exists public ${EXPECTED_NEXT_TABLE}

rm "$CONFIGURATION_FILE"
}

@test "Test interval change" {
Expand Down Expand Up @@ -228,7 +241,7 @@ EOF
assert_success
assert_output --partial "All partitions are correctly provisioned"

run list_existing_partitions "unittest" "public" ${TABLE}
run list_existing_partitions "public" ${TABLE}

local expected_monthly=$(cat <<EOF
public|test_interv_2024_12|2024-12-01|2025-01-01
Expand Down Expand Up @@ -261,9 +274,10 @@ public|test_interv_2025_w15|2025-04-07|2025-04-14
EOF
)

run list_existing_partitions "unittest" "public" ${TABLE}
run list_existing_partitions "public" ${TABLE}
assert_output "$expected_mix"

rm "$CONFIGURATION_FILE"
}

@test "Test provisioning with multiple partition sets in the configuration" {
Expand Down Expand Up @@ -301,7 +315,7 @@ public|table_unittest1_2025_02_01|2025-02-01|2025-02-02
public|table_unittest1_2025_02_02|2025-02-02|2025-02-03
EOF
)
run list_existing_partitions "unittest" "public" "table_unittest1"
run list_existing_partitions "public" "table_unittest1"
assert_output "$expected1"

local expected2=$(cat <<'EOF'
Expand All @@ -313,8 +327,10 @@ public|table_unittest2_2025_02_03|2025-02-03|2025-02-04
EOF
)

run list_existing_partitions "unittest" "public" "table_unittest2"
run list_existing_partitions "public" "table_unittest2"
assert_output "$expected2"

rm "$CONFIGURATION_FILE"
}

@test "Test that provisioning continues after an error on a partition set" {
Expand Down Expand Up @@ -361,8 +377,10 @@ public|${TABLE}_2025_02_03|2025-02-03|2025-02-04
EOF
)

run list_existing_partitions "unittest" "public" "${TABLE}"
run list_existing_partitions "public" "${TABLE}"
assert_output "$expected2"

rm "$CONFIGURATION_FILE"
}

@test "Test a timestamptz key with provisioning crossing a DST transition " {
Expand Down Expand Up @@ -392,7 +410,7 @@ public|${TABLE}_2025_w12|2025-03-17 00:00:00+01|2025-03-24 00:00:00+01
EOF
)

run list_existing_partitions "unittest" "public" ${TABLE}
run list_existing_partitions "public" ${TABLE}
assert_output "$expected_1"

# Now advance one week up to the end of March 2025
Expand All @@ -412,7 +430,9 @@ public|${TABLE}_2025_w12|2025-03-17 00:00:00+01|2025-03-24 00:00:00+01
public|${TABLE}_2025_w13|2025-03-24 00:00:00+01|2025-03-31 00:00:00+02
EOF
)
run list_existing_partitions "unittest" "public" ${TABLE}
run list_existing_partitions "public" ${TABLE}
assert_output "$expected_2"

rm "$CONFIGURATION_FILE"

}
10 changes: 8 additions & 2 deletions scripts/bats/40_cleanup.bats
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@ load 'test/libs/seeds'
load 'test/libs/sql'

setup_file() {
reset_database
init_database
}

teardown_file() {
drop_database
}

setup() {
Expand Down Expand Up @@ -63,6 +67,8 @@ EOF
for ((i=NEW_RETENTION +1; i<= INITIAL_PREPROVISIONED; i++));do
assert_table_not_exists public $(generate_daily_partition_name ${TABLE} ${i})
done

rm "$CONFIGURATION_FILE"
}

@test "Test that gaps in the partition set prevent any partition removal" {
Expand Down Expand Up @@ -117,7 +123,7 @@ EOF
expected+="public|${PARTS[i]}|${PARTS[i+1]}|${PARTS[i+2]}"
(( i+=3 ))
done
run list_existing_partitions "unittest" "public" "${TABLE}"
run list_existing_partitions "public" "${TABLE}"

assert_output "$expected"

Expand Down
2 changes: 1 addition & 1 deletion scripts/bats/configuration/template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ debug: true

log-format: text

connection-url: postgres://postgres:hackme@localhost/unittest
#connection-url: postgres://user:password@localhost/dbname
6 changes: 3 additions & 3 deletions scripts/bats/test/libs/partitions.bash
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ generate_daily_partition() {
local UPPER_BOUND=$(date -d "@$(( $(date +%s) + 86400 * $TIMEDELTA + 86400))" +"%Y-%m-%d")

local QUERY="CREATE TABLE ${TABLE_NAME} PARTITION OF ${PARENT_TABLE} FOR VALUES FROM ('${LOWER_BOUND}') TO ('${UPPER_BOUND}');"
execute_sql "${QUERY}"
execute_sql "${QUERY}" "$PPM_DATABASE"
}

generate_daily_partition_name() {
Expand All @@ -52,7 +52,7 @@ generate_daily_partition_name() {
}

generate_table_name() {
cat /dev/urandom | head -n 1 | base64 | tr -dc '[:alnum:]' | tr '[:upper:]' '[:lower:]' | cut -c -13 | sed -e 's/^[0-9]/a/g'
echo "tbl_"$(random_suffix)
}

# Generic method to create child partitions with specified ranges
Expand Down Expand Up @@ -85,5 +85,5 @@ create_partitions() {
done
sql_block="$sql_block
COMMIT;";
execute_sql_commands "$sql_block"
execute_sql_commands "$sql_block" "$PPM_DATABASE"
}
20 changes: 15 additions & 5 deletions scripts/bats/test/libs/seeds.bash
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
# Emit a random string suitable for unquoted database identifiers (lower case, ASCII)
# The result may start with a number.
random_suffix() {
head -c 12 /dev/urandom | base64 | tr -dc '[:alnum:]' | tr '[:upper:]' '[:lower:]'
}

init_database() {
QUERY="CREATE DATABASE unittest;"
local dbname="ppm_test_"$(random_suffix)
QUERY="CREATE DATABASE \"$dbname\" ;"
execute_sql "${QUERY}" postgres
export PPM_DATABASE="$dbname"
export PGDATABASE="$dbname"
}

drop_database() {
QUERY="set lock_timeout to '5s'; DROP DATABASE IF EXISTS unittest;"
QUERY="set lock_timeout to '5s'; DROP DATABASE IF EXISTS \"$PPM_DATABASE\" ;"
execute_sql_commands "${QUERY}" postgres
unset PPM_DATABASE
}

reset_database() {
Expand All @@ -23,7 +33,7 @@ create_table_from_template() {
created_at DATE NOT NULL
) PARTITION BY RANGE (created_at);
EOQ
execute_sql "${QUERY}"
execute_sql "${QUERY}" "${PPM_DATABASE}"
}

create_table_uuid_range() {
Expand Down Expand Up @@ -61,8 +71,8 @@ generate_configuration_file() {

local FILENAME=$(mktemp).yaml
yq '. as $item ireduce ({}; . * $item )' "${CONFIGURATION_TEMPLATE_FILE}" "${TEMPORARY_FILE}" > "${FILENAME}"

echo $FILENAME
rm "${TEMPORARY_FILE}"
echo "$FILENAME"
}

# Return a common configuration
Expand Down
33 changes: 20 additions & 13 deletions scripts/bats/test/libs/sql.bash
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ execute_sql() {
local SQL_COMMAND=$1
local DATABASE=$2

if [ -z ${DATABASE} ]; then
if [ -z "$DATABASE" ]; then
psql --echo-all --echo-errors -c "${SQL_COMMAND}"
else
psql --echo-all --echo-errors --dbname="${DATABASE}" -c "${SQL_COMMAND}"
Expand All @@ -13,7 +13,7 @@ execute_sql_file() {
local SQL_FILE=$1
local DATABASE=$2

if [ -z ${DATABASE} ]; then
if [ -z "$DATABASE" ]; then
psql --echo-all --echo-errors -f "${SQL_FILE}"
else
psql --echo-all --echo-errors --dbname="${DATABASE}" -f "${SQL_FILE}"
Expand All @@ -25,16 +25,23 @@ execute_sql_commands() {
# $1: sql commands
# $2: dbname (if unset, psql defaults to $PGDATABASE and then $USER)

psql --tuples-only --no-align --quiet "$2" <<EOSQL
local command="psql --tuples-only --no-align --quiet"

if [ ! -z "$2" ]; then
command="$command --dbname=$2"
fi

$command <<EOSQL
$1
EOSQL
}

list_existing_partitions() {
local DATABASE=$1
local PARENT_SCHEMA=$2
local PARENT_TABLE=$3
PGDATABASE="$DATABASE" psql --tuples-only --no-align --quiet -v parent_schema=${PARENT_SCHEMA} -v parent_table=${PARENT_TABLE} <<'EOSQL'
# mandatory arguments
local PARENT_SCHEMA=$1
local PARENT_TABLE=$2

psql --tuples-only --no-align --quiet --dbname="$PPM_DATABASE" -v parent_schema=${PARENT_SCHEMA} -v parent_table=${PARENT_TABLE} <<'EOSQL'
WITH parts as (
SELECT
relnamespace::regnamespace as schema,
Expand All @@ -58,8 +65,8 @@ EOSQL
}

assert_table_exists() {
local SCHEMA=$1
local TABLE=$2
local SCHEMA="$1"
local TABLE="$2"

local QUERY="SELECT EXISTS (
SELECT
Expand All @@ -68,15 +75,15 @@ assert_table_exists() {
AND table_name = '${TABLE}'
);"

run psql --tuples-only --no-align -c "${QUERY}"
run psql --dbname="$PPM_DATABASE" --tuples-only --no-align -c "${QUERY}"

assert_success
assert_output t
}

assert_table_not_exists() {
local SCHEMA=$1
local TABLE=$2
local SCHEMA="$1"
local TABLE="$2"

local QUERY="SELECT EXISTS (
SELECT
Expand All @@ -85,7 +92,7 @@ assert_table_not_exists() {
AND table_name = '${TABLE}'
);"

run psql --tuples-only --no-align -c "${QUERY}"
run psql --dbname="$PPM_DATABASE" --tuples-only --no-align -c "${QUERY}"

assert_success
assert_output f
Expand Down
Loading