add integration tests

.github/workflows/dockertests.yml

@@ -93,6 +93,7 @@ jobs:
                    'make TEST="pg_wale_compatibility_test" pg_integration_test',
                    'make TEST="pg_delete_before_permanent_full_test" pg_integration_test',
                    'make TEST="pg_delete_before_permanent_delta_test" pg_integration_test',
+                   'make TEST="pg_delete_target_test" pg_integration_test',
                    'make mongo_test',
                    'make MONGO_VERSION="4.4.3" MONGO_MAJOR="4.4" mongo_features',
                    'make MONGO_VERSION="4.2.12" MONGO_MAJOR="4.2" mongo_features',

docker-compose.yml

@@ -46,6 +46,7 @@ services:
       &&  mkdir -p /export/deletebeforepermanentdeltabucket
       &&  mkdir -p /export/deletewithoutconfirm
       &&  mkdir -p /export/deleteendtoendbucket
+      &&  mkdir -p /export/deletetargetbucket
       &&  mkdir -p /export/mysqlfullxtrabackupbucket
       &&  mkdir -p /export/mysqlfullmysqldumpbucket
       &&  mkdir -p /export/mysqlbinlogpushfetchbucket
@@ -268,6 +269,17 @@ services:
     links:
       - s3
 
+  pg_delete_target_test:
+    build:
+      dockerfile: docker/pg_tests/Dockerfile_delete_target_test
+      context: .
+    image: wal-g/delete_target_test
+    container_name: wal-g_pg_delete_target_test
+    depends_on:
+      - s3
+    links:
+      - s3
+
   pg_ghost_table_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_ghost_table_test

docker/pg_tests/Dockerfile_delete_target_test

@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_target_test.sh"

docker/pg_tests/scripts/configs/delete_target_test_config.json

@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "1",
+"WALE_S3_PREFIX": "s3://deletetargetbucket",
+"WALG_USE_WAL_DELTA": "true"

docker/pg_tests/scripts/tests/delete_target_test.sh

@@ -0,0 +1,51 @@
+#!/bin/sh
+set -e -x
+  CONFIG_FILE="/tmp/configs/delete_target_test_config.json"
+COMMON_CONFIG="/tmp/configs/common_config.json"
+TMP_CONFIG="/tmp/configs/tmp_config.json"
+cat ${CONFIG_FILE} > ${TMP_CONFIG}
+echo "," >> ${TMP_CONFIG}
+cat ${COMMON_CONFIG} >> ${TMP_CONFIG}
+/tmp/scripts/wrap_config_file.sh ${TMP_CONFIG}
+
+/usr/lib/postgresql/10/bin/initdb ${PGDATA}
+
+echo "archive_mode = on" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g --config=${TMP_CONFIG} wal-push %p'" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_timeout = 600" >> /var/lib/postgresql/10/main/postgresql.conf
+
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+
+/tmp/scripts/wait_while_pg_not_ready.sh
+
+wal-g --config=${TMP_CONFIG} delete everything FORCE --confirm
+
+for i in 1 2 3
+do
+    pgbench -i -s 1 postgres &
+    sleep 1
+    wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+done
+
+lines_before_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 1 > /tmp/list_before_delete
+
+FULL_BACKUP=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'NR==2{print $1}')
+
+wal-g --config=${TMP_CONFIG} delete target ${FULL_BACKUP} --confirm
+
+lines_after_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 1 > /tmp/list_after_delete
+
+if [ $(($lines_before_delete-2)) -ne $lines_after_delete ];
+then
+    echo $(($lines_before_delete-2)) > /tmp/before_delete
+    echo $lines_after_delete > /tmp/after_delete
+    echo "Wrong number of deleted lines"
+    diff /tmp/before_delete /tmp/after_delete
+fi
+
+
+diff /tmp/list_before_delete /tmp/list_after_delete
+/tmp/scripts/drop_pg.sh
+rm ${TMP_CONFIG}