Add integration test

.github/workflows/dockertests.yml

@@ -94,6 +94,7 @@ jobs:
                    'make TEST="pg_delete_before_permanent_full_test" pg_integration_test',
                    'make TEST="pg_delete_before_permanent_delta_test" pg_integration_test',
                    'make TEST="pg_delete_target_test" pg_integration_test',
+                   'make TEST="pg_delete_target_delta_test" pg_integration_test',
                    'make mongo_test',
                    'make MONGO_VERSION="4.4.3" MONGO_MAJOR="4.4" mongo_features',
                    'make MONGO_VERSION="4.2.12" MONGO_MAJOR="4.2" mongo_features',

docker-compose.yml

@@ -280,6 +280,17 @@ services:
     links:
       - s3
 
+  pg_delete_target_delta_test:
+    build:
+      dockerfile: docker/pg_tests/Dockerfile_delete_target_delta_test
+      context: .
+    image: wal-g/delete_target_delta_test
+    container_name: wal-g_pg_delete_target_delta_test
+    depends_on:
+      - s3
+    links:
+      - s3
+
   pg_ghost_table_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_ghost_table_test

docker/pg_tests/Dockerfile_delete_target_delta_test

@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_target_delta_test.sh"

docker/pg_tests/scripts/configs/delete_target_delta_test_config.json

@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "100",
+"WALE_S3_PREFIX": "s3://deletetargetbucket",
+"WALG_USE_WAL_DELTA": "true"

docker/pg_tests/scripts/tests/delete_target_delta_test.sh

@@ -0,0 +1,75 @@
+#!/bin/sh
+set -e -x
+  CONFIG_FILE="/tmp/configs/delete_target_delta_test_config.json"
+COMMON_CONFIG="/tmp/configs/common_config.json"
+TMP_CONFIG="/tmp/configs/tmp_config.json"
+cat ${CONFIG_FILE} > ${TMP_CONFIG}
+echo "," >> ${TMP_CONFIG}
+cat ${COMMON_CONFIG} >> ${TMP_CONFIG}
+/tmp/scripts/wrap_config_file.sh ${TMP_CONFIG}
+
+/usr/lib/postgresql/10/bin/initdb ${PGDATA}
+
+echo "archive_mode = on" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g --config=${TMP_CONFIG} wal-push %p'" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_timeout = 600" >> /var/lib/postgresql/10/main/postgresql.conf
+
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+
+/tmp/scripts/wait_while_pg_not_ready.sh
+
+wal-g --config=${TMP_CONFIG} delete everything FORCE --confirm
+
+# create one base backup and one increment
+for i in 1 2
+do
+    pgbench -i -s 1 postgres &
+    sleep 1
+    wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+done
+
+# remember the name of the first increment
+FIRST_INCREMENT=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'END {print $1}')
+
+# make the second full backup
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --full
+
+# make the increment from the second full backup
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+
+# remember the backup-list output with two full backups and two increments.
+# later in the test we create new backups which should be deleted so lists should be identical
+lines_before_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 4 > /tmp/list_before_delete
+
+# now make increment from the FIRST_INCREMENT, which will be deleted later
+INCREMENT_TO_DELETE=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'END {print $1}')
+
+# make the increment from the INCREMENT_TO_DELETE
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${INCREMENT_TO_DELETE}
+
+# make one more increment from the INCREMENT_TO_DELETE
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${INCREMENT_TO_DELETE}
+
+# delete the INCREMENT_TO_DELETE, should leave only the first full backup w/ first increment and the second full backup w/ first increment
+wal-g --config=${TMP_CONFIG} delete target ${FIRST_INCREMENT} --confirm
+
+lines_after_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 4 > /tmp/list_after_delete
+
+if [ $(($lines_before_delete)) -ne $lines_after_delete ];
+then
+    echo $(($lines_before_delete)) > /tmp/before_delete
+    echo $lines_after_delete > /tmp/after_delete
+    echo "Wrong number of deleted lines"
+    diff /tmp/before_delete /tmp/after_delete
+fi
+
+
+diff /tmp/list_before_delete /tmp/list_after_delete
+/tmp/scripts/drop_pg.sh
+rm ${TMP_CONFIG}