Add support for custom format

Author: itbm

Dockerfile

@@ -41,8 +41,10 @@ ENV DELETE_OLDER_THAN **None**
 ENV BACKUP_FILE **None**
 ENV CREATE_DATABASE no
 ENV DROP_DATABASE no
+ENV USE_CUSTOM_FORMAT no
 ENV COMPRESSION_CMD 'gzip'
 ENV DECOMPRESSION_CMD 'gunzip -c'
+ENV PARALLEL_JOBS 1
 
 ADD run.sh run.sh
 ADD backup.sh backup.sh

README.md

@@ -90,10 +90,12 @@ spec:
 | S3_ENDPOINT          |           |          | The AWS Endpoint URL, for S3 Compliant APIs such as [minio](https://minio.io)                                            |
 | S3_S3V4              | no        |          | Set to `yes` to enable AWS Signature Version 4, required for [minio](https://minio.io) servers                           |
 | SCHEDULE             |           |          | Backup schedule time, see explainatons below                                                                             |
-| ENCRYPTION_PASSWORD  |           |          | Password to encrypt/decrypt the backup                                                                                  |
+| ENCRYPTION_PASSWORD  |           |          | Password to encrypt/decrypt the backup                                                                                   |
 | DELETE_OLDER_THAN    |           |          | Delete old backups, see explanation and warning below                                                                    |
-| COMPRESSION_CMD      | gzip      |          | Command used to compress the backup (e.g. `pigz` for parallel compression)                                              |
-| DECOMPRESSION_CMD    | gunzip -c |          | Command used to decompress the backup (e.g. `pigz -dc` for parallel decompression)                                      |
+| USE_CUSTOM_FORMAT    | no        |          | Use PostgreSQL's custom format (-Fc) instead of plain text with compression                                              |
+| COMPRESSION_CMD      | gzip      |          | Command used to compress the backup (e.g. `pigz` for parallel compression) - ignored when USE_CUSTOM_FORMAT=yes          |
+| DECOMPRESSION_CMD    | gunzip -c |          | Command used to decompress the backup (e.g. `pigz -dc` for parallel decompression) - ignored when USE_CUSTOM_FORMAT=yes  |
+| PARALLEL_JOBS        | 1         |          | Number of parallel jobs for pg_restore when using custom format backups                                                  |
 | BACKUP_FILE          |           | Y*       | Required for restore. The path to the backup file in S3, format: S3_PREFIX/filename                                      |
 | CREATE_DATABASE      | no        |          | For restore: Set to `yes` to create the database if it doesn't exist                                                     |
 | DROP_DATABASE        | no        |          | For restore: Set to `yes` to drop the database before restoring (caution: destroys existing data). Use with CREATE_DATABASE=yes to recreate it |
@@ -114,12 +116,36 @@ WARNING: this will delete all files in the S3_PREFIX path, not just those create
 
 You can additionally set the `ENCRYPTION_PASSWORD` environment variable like `-e ENCRYPTION_PASSWORD="superstrongpassword"` to encrypt the backup. The restore process will automatically detect encrypted backups and decrypt them when the `ENCRYPTION_PASSWORD` environment variable is set correctly. It can be manually decrypted using `openssl aes-256-cbc -d -in backup.sql.gz.enc -out backup.sql.gz`.
 
-### Compression Options
+### Backup Format and Compression Options
 
-By default, backups are compressed with `gzip` and decompressed with `gunzip -c`. For improved performance on multi-core systems, you can use `pigz` (parallel gzip) instead:
+There are two options for backup format:
+
+1. **Plain text format with compression** (default):
+   - Uses plain SQL text output compressed with gzip/pigz
+   - Standard and widely compatible
+
+2. **PostgreSQL custom format**:
+   - Enable with `-e USE_CUSTOM_FORMAT=yes`
+   - Significantly faster than plain text format
+   - Produces smaller backup files (built-in compression)
+   - Supports parallel restoration for faster restores
+   - Allows selective table/schema restoration
+   - Recommended for larger databases
+
+For plain text format, backups are compressed with `gzip` by default. For improved performance on multi-core systems, you can use `pigz` (parallel gzip) instead:
 
 ```sh
 $ docker run ... -e COMPRESSION_CMD=pigz ... itbm/postgres-backup-s3
 
 $ docker run ... -e DECOMPRESSION_CMD="pigz -dc" ... itbm/postgres-backup-s3
 ```
+
+When using custom format with parallel restore:
+
+```sh
+$ docker run ... -e USE_CUSTOM_FORMAT=yes ... itbm/postgres-backup-s3
+
+$ docker run ... -e PARALLEL_JOBS=4 -e BACKUP_FILE=backup/dbname_0000-00-00T00:00:00Z.dump ... itbm/postgres-backup-s3
+```
+
+Note: Custom format is not available when using `POSTGRES_DATABASE=all` as pg_dumpall does not support this format.

backup.sh

@@ -60,15 +60,23 @@ POSTGRES_HOST_OPTS="-h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER $POSTG
 
 echo "Creating dump of ${POSTGRES_DATABASE} database from ${POSTGRES_HOST}..."
 
-SRC_FILE=dump.sql.gz
-DEST_FILE=${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz
-
-COMPRESSION_CMD=${COMPRESSION_CMD:-gzip}
-
-if [ "${POSTGRES_DATABASE}" == "all" ]; then
-  pg_dumpall $POSTGRES_HOST_OPTS | $COMPRESSION_CMD > $SRC_FILE
+if [ "$USE_CUSTOM_FORMAT" = "yes" ]; then
+  SRC_FILE=dump.dump
+  DEST_FILE=${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").dump
+  if [ "${POSTGRES_DATABASE}" == "all" ]; then
+    echo "ERROR: Custom format (-Fc) is not supported with pg_dumpall."
+    exit 1
+  else
+    pg_dump -Fc $POSTGRES_HOST_OPTS $POSTGRES_DATABASE > $SRC_FILE
+  fi
 else
-  pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | $COMPRESSION_CMD > $SRC_FILE
+  SRC_FILE=dump.sql.gz
+  DEST_FILE=${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz
+  if [ "${POSTGRES_DATABASE}" == "all" ]; then
+    pg_dumpall $POSTGRES_HOST_OPTS | $COMPRESSION_CMD > $SRC_FILE
+  else
+    pg_dump $POSTGRES_HOST_OPTS $POSTGRES_DATABASE | $COMPRESSION_CMD > $SRC_FILE
+  fi
 fi
 
 

restore.sh

@@ -127,16 +127,28 @@ if [ "${CREATE_DATABASE}" = "yes" ]; then
 fi
 
 if [[ "$DOWNLOAD_PATH" == *.sql.gz ]]; then
-  DECOMPRESSION_CMD=${DECOMPRESSION_CMD:-"gunzip -c"}
   if [ "${POSTGRES_DATABASE}" == "all" ]; then
     echo "Restoring all databases"
     $DECOMPRESSION_CMD $DOWNLOAD_PATH | psql $POSTGRES_HOST_OPTS -d postgres
   else
     echo "Restoring database ${POSTGRES_DATABASE}"
     $DECOMPRESSION_CMD $DOWNLOAD_PATH | psql $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE
   fi
+elif [[ "$DOWNLOAD_PATH" == *.dump ]]; then
+  if [ "${POSTGRES_DATABASE}" == "all" ]; then
+    echo "ERROR: Custom format backup cannot be used to restore all databases."
+    exit 1
+  else
+    echo "Restoring database ${POSTGRES_DATABASE} from custom format"
+    if [ "$PARALLEL_JOBS" -gt 1 ]; then
+      echo "Using parallel restore with $PARALLEL_JOBS jobs"
+      pg_restore -j $PARALLEL_JOBS $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE $DOWNLOAD_PATH
+    else
+      pg_restore $POSTGRES_HOST_OPTS -d $POSTGRES_DATABASE $DOWNLOAD_PATH
+    fi
+  fi
 else
-  echo "ERROR: Unsupported backup format. Expected *.sql.gz file."
+  echo "ERROR: Unsupported backup format. Expected *.sql.gz or *.dump file."
   exit 1
 fi