From f5e9b13606269034e512d5d5f0b8d079dc1f7439 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Thu, 27 Jul 2023 08:13:42 +0200 Subject: [PATCH 01/26] DO-101 Initial commit --- Dockerfile | 45 +++++++++++++++++++++++++++++++++++++++++++++ README.md | 39 ++++++++++++++++++++++++++++++++++++++- backup.sh | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 backup.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..a17acff --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ +# syntax=docker/dockerfile:1.2 + +FROM debian:stable as base + +MAINTAINER NumDes + +ARG GOCRONVER=v0.0.10 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + curl \ + ca-certificates \ + postgresql-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && curl --fail --retry 4 --retry-all-errors -o /usr/local/bin/go-cron.gz -L https://github.com/prodrigestivill/go-cron/releases/download/$GOCRONVER/go-cron-linux-amd64.gz \ + && gzip -vnd /usr/local/bin/go-cron.gz && chmod a+x /usr/local/bin/go-cron + +RUN curl --location --output /usr/local/bin/mcli "https://dl.min.io/client/mc/release/linux-amd64/mc" && \ + chmod +x /usr/local/bin/mcli +RUN mcli -v + +ENV POSTGRES_DB="**None**" \ + POSTGRES_HOST="**None**" \ + POSTGRES_PORT=5432 \ + POSTGRES_USER="**None**" \ + POSTGRES_PASSWORD="**None**" \ + POSTGRES_EXTRA_OPTS="--blobs" \ +# SCHEDULE="@daily" \ + SCHEDULE="@every 7m" \ + BACKUP_SUFFIX="tar.gz" \ + HEALTHCHECK_PORT=8080 \ + S3_ACCESS_KEY_ID="**None**" \ + S3_SECRET_ACCESS_KEY="**None**" \ + S3_BUCKET="**None**" \ + S3_ENDPOINT="**None**" + +COPY backup.sh /backup.sh +RUN chmod +x backup.sh + +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["exec /usr/local/bin/go-cron -s \"$SCHEDULE\" -p \"$HEALTHCHECK_PORT\" -- /backup.sh"] + +HEALTHCHECK --interval=5m --timeout=3s \ + CMD curl -f "http://localhost:$HEALTHCHECK_PORT/" || exit 1 diff --git a/README.md b/README.md index 0624fb5..bb2dbfc 100644 --- a/README.md +++ b/README.md @@ -4,4 +4,41 @@ Docker image for universal postgres backups # Roadmap - [ ] Add support for S3 - [ ] Add retention policy settings by env vars -- [ ] Notify about backup status by HTTP-request \ No newline at end of file +- [ ] Notify about backup status by HTTP-request + +# Docker build +``` +docker build . -t pg-backups:0.0.1 +``` + +# How to backup manually +``` +docker run --rm -it \ + -e POSTGRES_HOST="FQDN-OR-IP" \ + -e POSTGRES_DB="DB-NAME" \ + -e POSTGRES_USER="DB-USER" \ + -e POSTGRES_PASSWORD="PASS" \ + -e S3_ENDPOINT=http://YOUR-S3 \ + -e S3_ACCESS_KEY_ID="KEY-ID" \ + -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ + -e S3_BUCKET="BUCKET-NAME" \ + --entrypoint /bin/bash \ + IMAGE-NAME:tag +``` +``` +# ./backup.sh +``` + +# How to backup using `go-cron` +``` +docker run -d \ + -e POSTGRES_HOST="FQDN-OR-IP" \ + -e POSTGRES_DB="DB-NAME" \ + -e POSTGRES_USER="DB-USER" \ + -e POSTGRES_PASSWORD="PASS" \ + -e S3_ENDPOINT=http://YOUR-S3 \ + -e S3_ACCESS_KEY_ID="KEY-ID" \ + -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ + -e S3_BUCKET="BUCKET-NAME" \ + IMAGE-NAME:tag +``` diff --git a/backup.sh b/backup.sh new file mode 100644 index 0000000..d5ad5af --- /dev/null +++ b/backup.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' + +# Will be name of directory in backet dd-mm-yyyy_hh-mm-ss + timestamp=$(date +%d-%m-%Y_%H-%M-%S) + +# Export stuff + export PGPASSWORD=$POSTGRES_PASSWORD + +# Will create base backup + echo "Creating backup of $POSTGRES_DB database..." + pg_dump --username $POSTGRES_USER \ + -h $POSTGRES_HOST \ + -p $POSTGRES_PORT \ + -d $POSTGRES_DB \ + $POSTGRES_EXTRA_OPTS \ + > $POSTGRES_DB.sql +# Do compression + tar -czvf $POSTGRES_DB.$BACKUP_SUFFIX $POSTGRES_DB.sql + +# Set S3 connection configuration + mcli alias set backup $S3_ENDPOINT $S3_ACCESS_KEY_ID $S3_SECRET_ACCESS_KEY + +# Create the bucket + mcli mb backup/$S3_BUCKET + mcli cp $POSTGRES_DB.$BACKUP_SUFFIX backup/$S3_BUCKET/$timestamp/$POSTGRES_DB.$BACKUP_SUFFIX + +# Do nettoyage + echo "Maid is here... Doing cleaning..." + rm -f $POSTGRES_DB.* From fe5105e4a9ddbf1734cb04b4d03c23544e449ff5 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Thu, 27 Jul 2023 08:20:44 +0200 Subject: [PATCH 02/26] DO-101 Initial commit --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a17acff..5c5ff87 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,7 @@ ENV POSTGRES_DB="**None**" \ POSTGRES_USER="**None**" \ POSTGRES_PASSWORD="**None**" \ POSTGRES_EXTRA_OPTS="--blobs" \ -# SCHEDULE="@daily" \ - SCHEDULE="@every 7m" \ + SCHEDULE="@daily" \ BACKUP_SUFFIX="tar.gz" \ HEALTHCHECK_PORT=8080 \ S3_ACCESS_KEY_ID="**None**" \ From 754d5b896c5099fe4885b1ded6af21b2f35a0061 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 10:00:40 +0200 Subject: [PATCH 03/26] DO-101 Add GitHub Actions flow --- .github/workflows/main.yaml | 106 ++++++++++++++++++++++++++++++++++++ README.md | 7 ++- 2 files changed, 110 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/main.yaml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..e5b3576 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,106 @@ +name: Publish Docker image + +on: + workflow_dispatch: + push: + branches: + - 'main' + - 'master' + tags: + - 'v*' + pull_request: + branches: + - 'main' + - 'master' + +jobs: + push_to_registry: + name: Push Docker image to Docker Hub + runs-on: ubuntu-latest + steps: + - name: Check out the repo + uses: actions/checkout@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: thetips4you/testjodejsapp + + - name: Build and push Docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + +name: Docker Image CI + +on: + workflow_dispatch: + push: + branches: + - 'main' + - 'master' + tags: + - 'v*' + pull_request: + branches: + - 'main' + - 'master' + +env: + REGISTRY: docker.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + # Login against a Docker registry + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/README.md b/README.md index bb2dbfc..02d3d5b 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,13 @@ Docker image for universal postgres backups - [ ] Add retention policy settings by env vars - [ ] Notify about backup status by HTTP-request -# Docker build +## Docker build ``` docker build . -t pg-backups:0.0.1 ``` -# How to backup manually +# Usage +## Backup manually: ``` docker run --rm -it \ -e POSTGRES_HOST="FQDN-OR-IP" \ @@ -29,7 +30,7 @@ docker run --rm -it \ # ./backup.sh ``` -# How to backup using `go-cron` +## Backup using `go-cron` ``` docker run -d \ -e POSTGRES_HOST="FQDN-OR-IP" \ From 7015ab8c43375185fbaf8c13371d56e3893a924f Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 10:02:19 +0200 Subject: [PATCH 04/26] DO-101 Add GitHub Actions flow --- .github/workflows/main.yaml | 44 ------------------------------------- 1 file changed, 44 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index e5b3576..960caa5 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,47 +1,3 @@ -name: Publish Docker image - -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'master' - tags: - - 'v*' - pull_request: - branches: - - 'main' - - 'master' - -jobs: - push_to_registry: - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 - with: - images: thetips4you/testjodejsapp - - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: . - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - - name: Docker Image CI on: From 196ff606e52a6ab084f555271a43964dfb63cfd5 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 10:06:01 +0200 Subject: [PATCH 05/26] DO-101 Add GitHub Actions flow --- .github/workflows/main.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 960caa5..3ea4a9d 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,17 +1,17 @@ name: Docker Image CI -on: - workflow_dispatch: - push: - branches: - - 'main' - - 'master' - tags: - - 'v*' - pull_request: - branches: - - 'main' - - 'master' +on: push + # workflow_dispatch: + # push: + # branches: + # - 'main' + # - 'master' + # tags: + # - 'v*' + # pull_request: + # branches: + # - 'main' + # - 'master' env: REGISTRY: docker.io From b57d29ae8643801060779f9d2da5601fa622f793 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 10:08:48 +0200 Subject: [PATCH 06/26] DO-101 Add GitHub Actions flow --- .github/workflows/main.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 3ea4a9d..b50bc9f 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -37,8 +37,8 @@ jobs: uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c with: registry: ${{ env.REGISTRY }} - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} # Extract metadata (tags, labels) for Docker # https://github.com/docker/metadata-action From 26df47873b34c70132f3d9f227756e1eff1d29ef Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 12:08:54 +0200 Subject: [PATCH 07/26] DO-101 Add GitHub Actions flow --- .github/workflows/main.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index b50bc9f..9b697f3 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -1,17 +1,17 @@ name: Docker Image CI -on: push - # workflow_dispatch: - # push: - # branches: - # - 'main' - # - 'master' - # tags: - # - 'v*' - # pull_request: - # branches: - # - 'main' - # - 'master' +on: + workflow_dispatch: + push: + branches: + - 'main' + - 'master' + tags: + - 'v*' + pull_request: + branches: + - 'main' + - 'master' env: REGISTRY: docker.io From b2b66449d0802e2d660460d292ebc4190775bdec Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 12:42:47 +0200 Subject: [PATCH 08/26] DO-101 Add GitHub Actions flow --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 5c5ff87..f6136d4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,9 @@ FROM debian:stable as base MAINTAINER NumDes +LABEL org.opencontainers.image.vendor="Numerical Design LLC" +LABEL org.opencontainers.image.description="Docker image for universal postgres backups" + ARG GOCRONVER=v0.0.10 RUN apt-get update \ From 29a721bd1f5cd4a24756948780f8896f69be4d93 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 15:51:14 +0200 Subject: [PATCH 09/26] DO-101 Add webhook notification --- Dockerfile | 2 ++ README.md | 11 +++++++++++ backup.sh | 10 +++++++--- hooks/00-webhook.sh | 13 +++++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100755 hooks/00-webhook.sh diff --git a/Dockerfile b/Dockerfile index f6136d4..cf058ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,6 +40,8 @@ ENV POSTGRES_DB="**None**" \ COPY backup.sh /backup.sh RUN chmod +x backup.sh +COPY hooks /hooks + ENTRYPOINT ["/bin/sh", "-c"] CMD ["exec /usr/local/bin/go-cron -s \"$SCHEDULE\" -p \"$HEALTHCHECK_PORT\" -- /backup.sh"] diff --git a/README.md b/README.md index 02d3d5b..ad4165b 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ docker run --rm -it \ -e S3_ACCESS_KEY_ID="KEY-ID" \ -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ -e S3_BUCKET="BUCKET-NAME" \ + -e WEBHOOK_URL=http://webhook \ + -e TG_GROUP=point_to_notify_group \ + -e POSTGRES_PORT=if_not_5432 \ --entrypoint /bin/bash \ IMAGE-NAME:tag ``` @@ -41,5 +44,13 @@ docker run -d \ -e S3_ACCESS_KEY_ID="KEY-ID" \ -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ -e S3_BUCKET="BUCKET-NAME" \ + -e WEBHOOK_URL=http://webhook \ + -e TG_GROUP=point_to_notify_group \ + -e POSTGRES_PORT=if_not_5432 \ IMAGE-NAME:tag ``` + +## Variables + +TG_GROUP - Notifying group +WEBHOOK_URL - Notificator URL diff --git a/backup.sh b/backup.sh index d5ad5af..5fd809b 100644 --- a/backup.sh +++ b/backup.sh @@ -23,10 +23,14 @@ IFS=$'\n\t' # Set S3 connection configuration mcli alias set backup $S3_ENDPOINT $S3_ACCESS_KEY_ID $S3_SECRET_ACCESS_KEY -# Create the bucket - mcli mb backup/$S3_BUCKET - mcli cp $POSTGRES_DB.$BACKUP_SUFFIX backup/$S3_BUCKET/$timestamp/$POSTGRES_DB.$BACKUP_SUFFIX +# Create the bucket (Only enable if neccessary) +# mcli mb backup/$S3_BUCKET + mcli cp $POSTGRES_DB.$BACKUP_SUFFIX backup/$S3_BUCKET/$POSTGRES_DB/$timestamp/$POSTGRES_DB.$BACKUP_SUFFIX # Do nettoyage echo "Maid is here... Doing cleaning..." rm -f $POSTGRES_DB.* + +# Do anounce + txt="Backuped successfully to $S3_ENDPOINT/$S3_BUCKET/$timestamp/$POSTGRES_DB.$BACKUP_SUFFIX" + hooks/00-webhook.sh $txt diff --git a/hooks/00-webhook.sh b/hooks/00-webhook.sh new file mode 100755 index 0000000..751d808 --- /dev/null +++ b/hooks/00-webhook.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -euo pipefail +IFS=$'\n\t' + +TXT="$1" + +curl -XPOST \ + --url "${WEBHOOK_URL}" \ + --header 'Content-Type: application/json' \ + --data "{\"key\": \"$TG_GROUP\", \"text\": \"$TXT\"}" \ + --max-time 10 \ + --retry 5 From 573db6dd11a867225e4cd5a5a45b94c6cf0d4667 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 15:55:47 +0200 Subject: [PATCH 10/26] DO-101 Add webhook notification --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ad4165b..78123d3 100644 --- a/README.md +++ b/README.md @@ -52,5 +52,7 @@ docker run -d \ ## Variables -TG_GROUP - Notifying group -WEBHOOK_URL - Notificator URL +| Name | Description | +|-------------|-------------------| +|TG_GROUP | Notifying group | +|WEBHOOK_URL | Notificator URL | From d9a2a99c88c0be2e4d241f15e85f3573534182c6 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 15:57:54 +0200 Subject: [PATCH 11/26] DO-101 Add webhook notification --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 78123d3..b500545 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,10 @@ Docker image for universal postgres backups # Roadmap -- [ ] Add support for S3 +- [X] Add support for S3 - [ ] Add retention policy settings by env vars - [ ] Notify about backup status by HTTP-request +- [ ] Add docker-compose example ## Docker build ``` From 1c2e0884ab7e67dc2b6aba2b0de61ba6d108c964 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 16:02:02 +0200 Subject: [PATCH 12/26] DO-101 Add webhook notification --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b500545..542abc7 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,7 @@ Docker image for universal postgres backups # Roadmap - [X] Add support for S3 +- [X] Add CI/CD to publish image to DockerHub - [ ] Add retention policy settings by env vars - [ ] Notify about backup status by HTTP-request - [ ] Add docker-compose example @@ -53,7 +54,9 @@ docker run -d \ ## Variables -| Name | Description | -|-------------|-------------------| -|TG_GROUP | Notifying group | -|WEBHOOK_URL | Notificator URL | +| Name | Description | +|-------------------|-------------------------------| +|TG_GROUP | Notifying group | +|WEBHOOK_URL | Notificator URL | +|DOCKERHUB_LOGIN | `Actions` Repository secret | +|DOCKERHUB_PASSWORD | `Actions` Repository secret | From 6121fe27f4906296e735d232dcfb3836e8a1b11b Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Fri, 28 Jul 2023 16:06:34 +0200 Subject: [PATCH 13/26] DO-101 proofreading README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 542abc7..ab6c496 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,13 @@ Docker image for universal postgres backups - [ ] Add docker-compose example ## Docker build -``` +```shell docker build . -t pg-backups:0.0.1 ``` # Usage ## Backup manually: -``` +```shell docker run --rm -it \ -e POSTGRES_HOST="FQDN-OR-IP" \ -e POSTGRES_DB="DB-NAME" \ @@ -31,12 +31,12 @@ docker run --rm -it \ --entrypoint /bin/bash \ IMAGE-NAME:tag ``` -``` +```shell # ./backup.sh ``` ## Backup using `go-cron` -``` +```shell docker run -d \ -e POSTGRES_HOST="FQDN-OR-IP" \ -e POSTGRES_DB="DB-NAME" \ From 97a948d28a731f22c92acd9d1b7126b5c23bfce3 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Sat, 29 Jul 2023 17:51:54 +0200 Subject: [PATCH 14/26] DO-101 proofreading README.md --- README.md | 15 ++++++++------- backup.sh | 42 +++++++++++++++++++++++++----------------- hooks/00-webhook.sh | 8 +++++--- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index ab6c496..81fc463 100644 --- a/README.md +++ b/README.md @@ -25,14 +25,15 @@ docker run --rm -it \ -e S3_ACCESS_KEY_ID="KEY-ID" \ -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ -e S3_BUCKET="BUCKET-NAME" \ - -e WEBHOOK_URL=http://webhook \ - -e TG_GROUP=point_to_notify_group \ + -e NOTIFICATION_URL=http://webhook \ + -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ --entrypoint /bin/bash \ IMAGE-NAME:tag ``` +To run backup, in active container shell call `backup.sh` script ```shell -# ./backup.sh +./backup.sh ``` ## Backup using `go-cron` @@ -46,8 +47,8 @@ docker run -d \ -e S3_ACCESS_KEY_ID="KEY-ID" \ -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ -e S3_BUCKET="BUCKET-NAME" \ - -e WEBHOOK_URL=http://webhook \ - -e TG_GROUP=point_to_notify_group \ + -e NOTIFICATION_URL=http://webhook \ + -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ IMAGE-NAME:tag ``` @@ -56,7 +57,7 @@ docker run -d \ | Name | Description | |-------------------|-------------------------------| -|TG_GROUP | Notifying group | -|WEBHOOK_URL | Notificator URL | +|TELEGRAM_CHAT_ID | Notifying group | +|NOTIFICATION_URL | Notifier URL | |DOCKERHUB_LOGIN | `Actions` Repository secret | |DOCKERHUB_PASSWORD | `Actions` Repository secret | diff --git a/backup.sh b/backup.sh index 5fd809b..c29d52a 100644 --- a/backup.sh +++ b/backup.sh @@ -1,36 +1,44 @@ #!/usr/bin/env bash +# +# Script made for backup PostgreSQL database from local (${POSTGRES_HOST}=127.0.0.1) +# or remote host. Created backup puts on S3 storage. On completion script calls +# notification script hooks/00-webhook.sh which sends report to given Telegram Chat +# +# TODO (siameseoriental) Implement error handling set -euo pipefail IFS=$'\n\t' # Will be name of directory in backet dd-mm-yyyy_hh-mm-ss - timestamp=$(date +%d-%m-%Y_%H-%M-%S) +timestamp="$(date date +%FT%T%Z)" # Export stuff - export PGPASSWORD=$POSTGRES_PASSWORD +export PGPASSWORD=${POSTGRES_PASSWORD} # Will create base backup - echo "Creating backup of $POSTGRES_DB database..." - pg_dump --username $POSTGRES_USER \ - -h $POSTGRES_HOST \ - -p $POSTGRES_PORT \ - -d $POSTGRES_DB \ - $POSTGRES_EXTRA_OPTS \ - > $POSTGRES_DB.sql +echo "Creating backup of ${POSTGRES_DB} database..." +pg_dump --username ${POSTGRES_USER} \ + -h ${POSTGRES_HOST} \ + -p ${POSTGRES_PORT} \ + -d ${POSTGRES_DB} \ + ${POSTGRES_EXTRA_OPTS} \ + > ${POSTGRES_DB}.sql # Do compression - tar -czvf $POSTGRES_DB.$BACKUP_SUFFIX $POSTGRES_DB.sql +tar -czvf "${POSTGRES_DB}.${BACKUP_SUFFIX} ${POSTGRES_DB}.sql" # Set S3 connection configuration - mcli alias set backup $S3_ENDPOINT $S3_ACCESS_KEY_ID $S3_SECRET_ACCESS_KEY +mcli alias set backup "${S3_ENDPOINT} ${S3_ACCESS_KEY_ID} ${S3_SECRET_ACCESS_KEY}" # Create the bucket (Only enable if neccessary) -# mcli mb backup/$S3_BUCKET - mcli cp $POSTGRES_DB.$BACKUP_SUFFIX backup/$S3_BUCKET/$POSTGRES_DB/$timestamp/$POSTGRES_DB.$BACKUP_SUFFIX +# mcli mb backup/${S3_BUCKET} +mcli cp ${POSTGRES_DB}.${BACKUP_SUFFIX} \ + backup/${S3_BUCKET}/${POSTGRES_DB}/${timestamp}/${POSTGRES_DB}.${BACKUP_SUFFIX} # Do nettoyage - echo "Maid is here... Doing cleaning..." - rm -f $POSTGRES_DB.* +echo "Maid is here... Doing cleaning..." +rm -f "${POSTGRES_DB}.*" # Do anounce - txt="Backuped successfully to $S3_ENDPOINT/$S3_BUCKET/$timestamp/$POSTGRES_DB.$BACKUP_SUFFIX" - hooks/00-webhook.sh $txt +txt="Backuped successfully to +${S3_ENDPOINT}/${S3_BUCKET}/${POSTGRES_DB}/${timestamp}/${POSTGRES_DB}.${BACKUP_SUFFIX}" +hooks/00-webhook.sh "${txt}" diff --git a/hooks/00-webhook.sh b/hooks/00-webhook.sh index 751d808..983e804 100755 --- a/hooks/00-webhook.sh +++ b/hooks/00-webhook.sh @@ -1,13 +1,15 @@ #!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat set -euo pipefail IFS=$'\n\t' -TXT="$1" +TXT="${1}" curl -XPOST \ - --url "${WEBHOOK_URL}" \ + --url "${NOTIFICATION_URL}" \ --header 'Content-Type: application/json' \ - --data "{\"key\": \"$TG_GROUP\", \"text\": \"$TXT\"}" \ + --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${TXT}\"}" \ --max-time 10 \ --retry 5 From 226723a10e65a2c9c29ec9a1d240ec15f3bd66d9 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Sat, 29 Jul 2023 18:23:10 +0200 Subject: [PATCH 15/26] DO-101 after shellcheck --- backup.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backup.sh b/backup.sh index c29d52a..a1413b1 100644 --- a/backup.sh +++ b/backup.sh @@ -17,12 +17,12 @@ export PGPASSWORD=${POSTGRES_PASSWORD} # Will create base backup echo "Creating backup of ${POSTGRES_DB} database..." -pg_dump --username ${POSTGRES_USER} \ - -h ${POSTGRES_HOST} \ - -p ${POSTGRES_PORT} \ - -d ${POSTGRES_DB} \ - ${POSTGRES_EXTRA_OPTS} \ - > ${POSTGRES_DB}.sql +pg_dump --username "${POSTGRES_USER}" \ + -h "${POSTGRES_HOST}" \ + -p "${POSTGRES_PORT}" \ + -d "${POSTGRES_DB}" \ + "${POSTGRES_EXTRA_OPTS}" \ + > "${POSTGRES_DB}".sql # Do compression tar -czvf "${POSTGRES_DB}.${BACKUP_SUFFIX} ${POSTGRES_DB}.sql" @@ -31,8 +31,8 @@ mcli alias set backup "${S3_ENDPOINT} ${S3_ACCESS_KEY_ID} ${S3_SECRET_ACCESS_KEY # Create the bucket (Only enable if neccessary) # mcli mb backup/${S3_BUCKET} -mcli cp ${POSTGRES_DB}.${BACKUP_SUFFIX} \ - backup/${S3_BUCKET}/${POSTGRES_DB}/${timestamp}/${POSTGRES_DB}.${BACKUP_SUFFIX} +mcli cp "${POSTGRES_DB}.${BACKUP_SUFFIX}" \ + backup/"${S3_BUCKET}/${POSTGRES_DB}/${timestamp}/${POSTGRES_DB}.${BACKUP_SUFFIX}" # Do nettoyage echo "Maid is here... Doing cleaning..." From 38dfa2ba6dad166e754670b9f6916351ff33862b Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Sat, 29 Jul 2023 22:32:19 +0200 Subject: [PATCH 16/26] DO-101 resolving discussions --- Dockerfile | 3 ++- README.md | 43 +++++++++++++++++++++++++------- backup.sh | 52 ++++++++++++++++++++++++++++++--------- hooks/00-webhook.sh | 15 ----------- hooks/external-webhook.sh | 13 ++++++++++ hooks/private-webhook.sh | 15 +++++++++++ 6 files changed, 105 insertions(+), 36 deletions(-) delete mode 100755 hooks/00-webhook.sh create mode 100755 hooks/external-webhook.sh create mode 100755 hooks/private-webhook.sh diff --git a/Dockerfile b/Dockerfile index cf058ce..560c57a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -35,7 +35,8 @@ ENV POSTGRES_DB="**None**" \ S3_ACCESS_KEY_ID="**None**" \ S3_SECRET_ACCESS_KEY="**None**" \ S3_BUCKET="**None**" \ - S3_ENDPOINT="**None**" + S3_ENDPOINT="**None**" \ + TELEGRAM_METHOD="private" COPY backup.sh /backup.sh RUN chmod +x backup.sh diff --git a/README.md b/README.md index 81fc463..4ecb9d5 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Docker image for universal postgres backups ## Docker build ```shell -docker build . -t pg-backups:0.0.1 +docker build . -t numdes/nd_postgres_backup:v*.*.* ``` # Usage @@ -29,7 +29,7 @@ docker run --rm -it \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ --entrypoint /bin/bash \ - IMAGE-NAME:tag + numdes/nd_postgres_backup:v*.*.* ``` To run backup, in active container shell call `backup.sh` script ```shell @@ -50,14 +50,39 @@ docker run -d \ -e NOTIFICATION_URL=http://webhook \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ - IMAGE-NAME:tag + numdes/nd_postgres_backup:v*.*.* ``` ## Variables -| Name | Description | -|-------------------|-------------------------------| -|TELEGRAM_CHAT_ID | Notifying group | -|NOTIFICATION_URL | Notifier URL | -|DOCKERHUB_LOGIN | `Actions` Repository secret | -|DOCKERHUB_PASSWORD | `Actions` Repository secret | +| Name | Description | +|-------------------|-----------------------------------------------------| +|TELEGRAM_METHOD | By default used `private` | +|TELEGRAM_CHAT_ID | Notifying group | +|NOTIFICATION_URL | Notifier URL | +|TELEGRAM_BOT_TOKEN | Only used when TELEGRAM_METHOD is set to `external` | +|DOCKERHUB_LOGIN | `Actions` Repository secret | +|DOCKERHUB_PASSWORD | `Actions` Repository secret | + +### TELEGRAM_METHOD environment variable + +Variable is used to select which notification method is going to be used. In case of usage +local Telegram bot variable must be set to `private` (default). If public Telegram API +going to be selected then `TELEGRAM_METHOD` must be set to `external` and in `docker ...` command need to replace: +``` +-e NOTIFICATION_URL=http://webhook \ +-e TELEGRAM_CHAT_ID=point_to_notify_group \ +``` +to +``` + -e TELEGRAM_METHOD=external \ + -e TELEGRAM_BOT_TOKEN='XXXXXXX:XXXXxxxxXXXXxxx' \ + -e TELEGRAM_CHAT_ID=000000000 \ +``` +- If TELEGRAM_METHOD variable is set to `private` `private-webhook.sh` will be executed +and notification processing will be passed to internal Telegram bot. Along with +`private` flag following variables come: `TELEGRAM_CHAT_ID`, `NOTIFICATION_URL` +- If TELEGRAM_METHOD variable is set to `external` `external-webhook.sh` will be executed +and notification processing will be passed to standard Telegram API URL. Along with +`external` flag following variables come: `TELEGRAM_CHAT_ID`, `TELEGRAM_BOT_TOKEN` +- If TELEGRAM_METHOD variable is set to anything else only `echo` will be used diff --git a/backup.sh b/backup.sh index a1413b1..9dffc3e 100644 --- a/backup.sh +++ b/backup.sh @@ -3,20 +3,20 @@ # Script made for backup PostgreSQL database from local (${POSTGRES_HOST}=127.0.0.1) # or remote host. Created backup puts on S3 storage. On completion script calls # notification script hooks/00-webhook.sh which sends report to given Telegram Chat -# -# TODO (siameseoriental) Implement error handling set -euo pipefail IFS=$'\n\t' # Will be name of directory in backet dd-mm-yyyy_hh-mm-ss -timestamp="$(date date +%FT%T%Z)" +timestamp="$(date +%F_%T)" # Export stuff export PGPASSWORD=${POSTGRES_PASSWORD} # Will create base backup -echo "Creating backup of ${POSTGRES_DB} database..." +echo "Creating backup of ${POSTGRES_DB} database. From ${POSTGRES_HOST} and port \ +is ${POSTGRES_PORT}. Username: ${POSTGRES_USER}. With following extra options: \ +${POSTGRES_EXTRA_OPTS}" pg_dump --username "${POSTGRES_USER}" \ -h "${POSTGRES_HOST}" \ -p "${POSTGRES_PORT}" \ @@ -24,21 +24,51 @@ pg_dump --username "${POSTGRES_USER}" \ "${POSTGRES_EXTRA_OPTS}" \ > "${POSTGRES_DB}".sql # Do compression -tar -czvf "${POSTGRES_DB}.${BACKUP_SUFFIX} ${POSTGRES_DB}.sql" +tar -czvf "${POSTGRES_DB}.${BACKUP_SUFFIX}" "${POSTGRES_DB}.sql" + +# Count file size +size_in_bytes="$(du -b "${POSTGRES_DB}.${BACKUP_SUFFIX}" | awk '{print $1}')" +if (( size_in_bytes < 1048576 )); then + file_measure=" Kb" + file_size="$((size_in_bytes / 1024))" + send_file_size="${file_size}${file_measure}" +elif (( size_in_bytes < 1073741824 )); then + file_measure=" Mb" + file_size="$((size_in_bytes / (1024 * 1024)))" + send_file_size="${file_size}${file_measure}" +else + file_measure=" Gb" + file_size="$((size_in_bytes / (1024 * 1024 * 1024)))" + send_file_size="${file_size}${file_measure}" +fi # Set S3 connection configuration -mcli alias set backup "${S3_ENDPOINT} ${S3_ACCESS_KEY_ID} ${S3_SECRET_ACCESS_KEY}" +mcli alias set backup "${S3_ENDPOINT}" "${S3_ACCESS_KEY_ID}" "${S3_SECRET_ACCESS_KEY}" + +# Declaring variables for informational purposes +copy_file_name="${POSTGRES_DB}.${BACKUP_SUFFIX}" +copy_path="${S3_BUCKET}/${POSTGRES_DB}/${timestamp}" +mcli_copy_path="${copy_path}/${copy_file_name}" +info_copy_path="${S3_ENDPOINT}/${copy_path}" +echo "Starting to copy ${copy_file_name} to ${info_copy_path}..." # Create the bucket (Only enable if neccessary) # mcli mb backup/${S3_BUCKET} -mcli cp "${POSTGRES_DB}.${BACKUP_SUFFIX}" \ - backup/"${S3_BUCKET}/${POSTGRES_DB}/${timestamp}/${POSTGRES_DB}.${BACKUP_SUFFIX}" +mcli cp "${copy_file_name}" backup/"${mcli_copy_path}" # Do nettoyage echo "Maid is here... Doing cleaning..." rm -f "${POSTGRES_DB}.*" # Do anounce -txt="Backuped successfully to -${S3_ENDPOINT}/${S3_BUCKET}/${POSTGRES_DB}/${timestamp}/${POSTGRES_DB}.${BACKUP_SUFFIX}" -hooks/00-webhook.sh "${txt}" +if [[ ${TELEGRAM_METHOD} == 'private' ]]; then + txt="Backed up ${copy_file_name} with file size: ${send_file_size} \ + to ${info_copy_path}" + hooks/private-webhook.sh "${txt}" +elif [[ ${TELEGRAM_METHOD} == 'external' ]]; then + txt="Backed up ${copy_file_name} with file size: ${send_file_size} \ + to ${info_copy_path}" + hooks/external-webhook.sh "${txt}" +else + echo "No notification methods selected" +fi diff --git a/hooks/00-webhook.sh b/hooks/00-webhook.sh deleted file mode 100755 index 983e804..0000000 --- a/hooks/00-webhook.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# -# Use http request to send notification to Telegram chat - -set -euo pipefail -IFS=$'\n\t' - -TXT="${1}" - -curl -XPOST \ - --url "${NOTIFICATION_URL}" \ - --header 'Content-Type: application/json' \ - --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${TXT}\"}" \ - --max-time 10 \ - --retry 5 diff --git a/hooks/external-webhook.sh b/hooks/external-webhook.sh new file mode 100755 index 0000000..9b1d98c --- /dev/null +++ b/hooks/external-webhook.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat through external Telegram API + +set -euo pipefail +IFS=$'\n\t' + +message_text="${1}" + +curl -s \ + --data "text=${message_text}" \ + --data "chat_id=${TELEGRAM_CHAT_ID}" \ + 'https://api.telegram.org/bot'${TELEGRAM_BOT_TOKEN}'/sendMessage' > /dev/null diff --git a/hooks/private-webhook.sh b/hooks/private-webhook.sh new file mode 100755 index 0000000..961ea16 --- /dev/null +++ b/hooks/private-webhook.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat through local Telegram bot + +set -euo pipefail +IFS=$'\n\t' + +message_text="${1}" + +curl -XPOST \ + --url "${NOTIFICATION_URL}" \ + --header 'Content-Type: application/json' \ + --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${message_text}\"}" \ + --max-time 10 \ + --retry 5 From 79a5b961906d38d7ec774021fffcc0fed795aa6f Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Sat, 29 Jul 2023 22:34:42 +0200 Subject: [PATCH 17/26] DO-101 proofreading README.md --- README.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 4ecb9d5..73f0497 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,15 @@ docker run -d \ Variable is used to select which notification method is going to be used. In case of usage local Telegram bot variable must be set to `private` (default). If public Telegram API -going to be selected then `TELEGRAM_METHOD` must be set to `external` and in `docker ...` command need to replace: +going to be selected then `TELEGRAM_METHOD` must be set to `external`. +- If TELEGRAM_METHOD variable is set to `private` `private-webhook.sh` will be executed +and notification processing will be passed to internal Telegram bot. Along with +`private` flag following variables come: `TELEGRAM_CHAT_ID`, `NOTIFICATION_URL` +- If TELEGRAM_METHOD variable is set to `external` `external-webhook.sh` will be executed +and notification processing will be passed to standard Telegram API URL. Along with +`external` flag following variables come: `TELEGRAM_CHAT_ID`, `TELEGRAM_BOT_TOKEN` + +In `docker ...` command need to replace: ``` -e NOTIFICATION_URL=http://webhook \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ @@ -78,11 +86,5 @@ to -e TELEGRAM_METHOD=external \ -e TELEGRAM_BOT_TOKEN='XXXXXXX:XXXXxxxxXXXXxxx' \ -e TELEGRAM_CHAT_ID=000000000 \ -``` -- If TELEGRAM_METHOD variable is set to `private` `private-webhook.sh` will be executed -and notification processing will be passed to internal Telegram bot. Along with -`private` flag following variables come: `TELEGRAM_CHAT_ID`, `NOTIFICATION_URL` -- If TELEGRAM_METHOD variable is set to `external` `external-webhook.sh` will be executed -and notification processing will be passed to standard Telegram API URL. Along with -`external` flag following variables come: `TELEGRAM_CHAT_ID`, `TELEGRAM_BOT_TOKEN` +``` - If TELEGRAM_METHOD variable is set to anything else only `echo` will be used From 360573ad305906e8c9a37512f4e787bbb322bed9 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Sat, 29 Jul 2023 22:48:10 +0200 Subject: [PATCH 18/26] DO-101 resolving forgotten discussion --- backup.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/backup.sh b/backup.sh index 9dffc3e..02699e7 100644 --- a/backup.sh +++ b/backup.sh @@ -23,11 +23,18 @@ pg_dump --username "${POSTGRES_USER}" \ -d "${POSTGRES_DB}" \ "${POSTGRES_EXTRA_OPTS}" \ > "${POSTGRES_DB}".sql + +# Declaring variables for informational purposes +copy_file_name="${POSTGRES_DB}.${BACKUP_SUFFIX}" +copy_path="${S3_BUCKET}/${POSTGRES_DB}/${timestamp}" +mcli_copy_path="${copy_path}/${copy_file_name}" +info_copy_path="${S3_ENDPOINT}/${copy_path}" + # Do compression -tar -czvf "${POSTGRES_DB}.${BACKUP_SUFFIX}" "${POSTGRES_DB}.sql" +tar -czvf "${copy_file_name}" "${POSTGRES_DB}.sql" # Count file size -size_in_bytes="$(du -b "${POSTGRES_DB}.${BACKUP_SUFFIX}" | awk '{print $1}')" +size_in_bytes="$(du -b "${copy_file_name}" | awk '{print $1}')" if (( size_in_bytes < 1048576 )); then file_measure=" Kb" file_size="$((size_in_bytes / 1024))" @@ -42,14 +49,12 @@ else send_file_size="${file_size}${file_measure}" fi + +echo "Backed up ${copy_file_name} with file size: ${send_file_size}" + # Set S3 connection configuration mcli alias set backup "${S3_ENDPOINT}" "${S3_ACCESS_KEY_ID}" "${S3_SECRET_ACCESS_KEY}" -# Declaring variables for informational purposes -copy_file_name="${POSTGRES_DB}.${BACKUP_SUFFIX}" -copy_path="${S3_BUCKET}/${POSTGRES_DB}/${timestamp}" -mcli_copy_path="${copy_path}/${copy_file_name}" -info_copy_path="${S3_ENDPOINT}/${copy_path}" echo "Starting to copy ${copy_file_name} to ${info_copy_path}..." # Create the bucket (Only enable if neccessary) From 26ae85d445c0ae4a79c6917f5ef05b77e7bebe8a Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 15:01:25 +0200 Subject: [PATCH 19/26] DO-101 resolving discussion --- Dockerfile | 2 -- README.md | 63 +++++++++++++++++++++------------ backup.sh | 63 ++++++++++++--------------------- hooks/external-webhook.sh | 13 ------- hooks/private-webhook.sh | 15 -------- hooks/send_private_notification | 22 ++++++++++++ hooks/send_telegram_message | 25 +++++++++++++ 7 files changed, 110 insertions(+), 93 deletions(-) delete mode 100755 hooks/external-webhook.sh delete mode 100755 hooks/private-webhook.sh create mode 100755 hooks/send_private_notification create mode 100755 hooks/send_telegram_message diff --git a/Dockerfile b/Dockerfile index 560c57a..a82a8da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,13 +30,11 @@ ENV POSTGRES_DB="**None**" \ POSTGRES_PASSWORD="**None**" \ POSTGRES_EXTRA_OPTS="--blobs" \ SCHEDULE="@daily" \ - BACKUP_SUFFIX="tar.gz" \ HEALTHCHECK_PORT=8080 \ S3_ACCESS_KEY_ID="**None**" \ S3_SECRET_ACCESS_KEY="**None**" \ S3_BUCKET="**None**" \ S3_ENDPOINT="**None**" \ - TELEGRAM_METHOD="private" COPY backup.sh /backup.sh RUN chmod +x backup.sh diff --git a/README.md b/README.md index 73f0497..022ff5f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Docker image for universal postgres backups - [X] Add support for S3 - [X] Add CI/CD to publish image to DockerHub - [ ] Add retention policy settings by env vars -- [ ] Notify about backup status by HTTP-request +- [X] Notify about backup status by HTTP-request - [ ] Add docker-compose example ## Docker build @@ -25,7 +25,7 @@ docker run --rm -it \ -e S3_ACCESS_KEY_ID="KEY-ID" \ -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ -e S3_BUCKET="BUCKET-NAME" \ - -e NOTIFICATION_URL=http://webhook \ + -e PRIVATE_NOTIFICATION_URL=http://webhook \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ --entrypoint /bin/bash \ @@ -47,44 +47,61 @@ docker run -d \ -e S3_ACCESS_KEY_ID="KEY-ID" \ -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ -e S3_BUCKET="BUCKET-NAME" \ - -e NOTIFICATION_URL=http://webhook \ + -e PRIVATE_NOTIFICATION_URL=http://webhook \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ + -e SCHEDULE=[Chosen_schedule][1] numdes/nd_postgres_backup:v*.*.* ``` +[1]: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*. ## Variables - +### `Gitlab Actions` *[variables](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository)*: | Name | Description | |-------------------|-----------------------------------------------------| -|TELEGRAM_METHOD | By default used `private` | -|TELEGRAM_CHAT_ID | Notifying group | -|NOTIFICATION_URL | Notifier URL | -|TELEGRAM_BOT_TOKEN | Only used when TELEGRAM_METHOD is set to `external` | |DOCKERHUB_LOGIN | `Actions` Repository secret | |DOCKERHUB_PASSWORD | `Actions` Repository secret | -### TELEGRAM_METHOD environment variable +### Notification environmental variables +| Name | Description | +|---------------------------|-----------------------------------------------------| +|TELEGRAM_CHAT_ID | Notifying group | +|PRIVATE_NOTIFICATION_URL | Private notifier URL | +|TELEGRAM_BOT_TOKEN | Only used to call Telegram's public API | + +### Environmental variables +| Name | Default | Description | +| | value | | +|------------------------| :------ |------------------------------------------------| +| POSTGRES_DB | - | Database name | +| POSTGRES_HOST | - | PostgreSQL IP address or hostname | +| POSTGRES_PORT | 5432 | Connection TCP port | +| POSTGRES_USER | - | Database user | +| POSTGRES_PASSWORD | - | Database user password | +| POSTGRES_EXTRA_OPTS | --blobs | Extra options `pg_dump` run | +| SCHEDULE | @daily | `go-cron` schedule. See *[this](https://github.com/siameseoriental/nd_postgres_backup/tree/DO-101#backup-using-go-cron)* | +| HEALTHCHECK_PORT | 8080 | Port listening for cron-schedule health check. | +| S3_ACCESS_KEY_ID | - | Key or username with RW access to bucket | +| S3_SECRET_ACCESS_KEY | - | Secret or password for `S3_ACCESS_KEY_ID` | +| S3_BUCKET | - | Name of bucket created for backups | +| S3_ENDPOINT | - | URL of S3 storage | + +### Notification selection + +It is possible to use either private Telegram bot if you have it or Telegram public API. -Variable is used to select which notification method is going to be used. In case of usage -local Telegram bot variable must be set to `private` (default). If public Telegram API -going to be selected then `TELEGRAM_METHOD` must be set to `external`. -- If TELEGRAM_METHOD variable is set to `private` `private-webhook.sh` will be executed -and notification processing will be passed to internal Telegram bot. Along with -`private` flag following variables come: `TELEGRAM_CHAT_ID`, `NOTIFICATION_URL` -- If TELEGRAM_METHOD variable is set to `external` `external-webhook.sh` will be executed -and notification processing will be passed to standard Telegram API URL. Along with -`external` flag following variables come: `TELEGRAM_CHAT_ID`, `TELEGRAM_BOT_TOKEN` +In scenario with private bot `PRIVATE_NOTIFICATION_URL` must be set alongside with `TELEGRAM_CHAT_ID`. + +In scenario with Telegram's public API `TELEGRAM_BOT_TOKEN` must be set as it is received (`Use this token to access the HTTP API:`) from `@BotFather` Telegram Bot. Variable `TELEGRAM_CHAT_ID` must be a proper Telegram ID of bot In `docker ...` command need to replace: ``` --e NOTIFICATION_URL=http://webhook \ --e TELEGRAM_CHAT_ID=point_to_notify_group \ + -e PRIVATE_NOTIFICATION_URL=http://webhook \ + -e TELEGRAM_CHAT_ID=point_to_notify_group \ ``` to ``` - -e TELEGRAM_METHOD=external \ -e TELEGRAM_BOT_TOKEN='XXXXXXX:XXXXxxxxXXXXxxx' \ -e TELEGRAM_CHAT_ID=000000000 \ -``` -- If TELEGRAM_METHOD variable is set to anything else only `echo` will be used +``` +- If `TELEGRAM_CHAT_ID` has a proper format (Only digits not less than 5 not more than 32) and `TELEGRAM_BOT_TOKEN` is set, script will try to send notification through Telegram's public API. diff --git a/backup.sh b/backup.sh index 02699e7..9cd4b4c 100644 --- a/backup.sh +++ b/backup.sh @@ -1,79 +1,62 @@ #!/usr/bin/env bash # # Script made for backup PostgreSQL database from local (${POSTGRES_HOST}=127.0.0.1) -# or remote host. Created backup puts on S3 storage. On completion script calls +# or remote host. Created backup stores in S3 storage. On completion script calls # notification script hooks/00-webhook.sh which sends report to given Telegram Chat -set -euo pipefail +set -euox pipefail IFS=$'\n\t' -# Will be name of directory in backet dd-mm-yyyy_hh-mm-ss +# Will be name of directory in backet yyyy-mm-dd_HH:MM:SS timestamp="$(date +%F_%T)" -# Export stuff export PGPASSWORD=${POSTGRES_PASSWORD} # Will create base backup echo "Creating backup of ${POSTGRES_DB} database. From ${POSTGRES_HOST} and port \ is ${POSTGRES_PORT}. Username: ${POSTGRES_USER}. With following extra options: \ ${POSTGRES_EXTRA_OPTS}" -pg_dump --username "${POSTGRES_USER}" \ - -h "${POSTGRES_HOST}" \ - -p "${POSTGRES_PORT}" \ - -d "${POSTGRES_DB}" \ +pg_dump --username="${POSTGRES_USER}" \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --dbname="${POSTGRES_DB}" \ "${POSTGRES_EXTRA_OPTS}" \ > "${POSTGRES_DB}".sql # Declaring variables for informational purposes -copy_file_name="${POSTGRES_DB}.${BACKUP_SUFFIX}" +copy_file_name="${POSTGRES_DB}.tar.gz" copy_path="${S3_BUCKET}/${POSTGRES_DB}/${timestamp}" mcli_copy_path="${copy_path}/${copy_file_name}" info_copy_path="${S3_ENDPOINT}/${copy_path}" # Do compression -tar -czvf "${copy_file_name}" "${POSTGRES_DB}.sql" +#tar -czvf "${copy_file_name}" "${POSTGRES_DB}.sql" +tar --create \ + --gzip \ + --verbose \ + --file "${copy_file_name}" \ + "${POSTGRES_DB}.sql" # Count file size -size_in_bytes="$(du -b "${copy_file_name}" | awk '{print $1}')" -if (( size_in_bytes < 1048576 )); then - file_measure=" Kb" - file_size="$((size_in_bytes / 1024))" - send_file_size="${file_size}${file_measure}" -elif (( size_in_bytes < 1073741824 )); then - file_measure=" Mb" - file_size="$((size_in_bytes / (1024 * 1024)))" - send_file_size="${file_size}${file_measure}" -else - file_measure=" Gb" - file_size="$((size_in_bytes / (1024 * 1024 * 1024)))" - send_file_size="${file_size}${file_measure}" -fi +send_file_size="$(ls -lh | grep "${copy_file_name}" | awk '{print $5}')" - -echo "Backed up ${copy_file_name} with file size: ${send_file_size}" +echo "Created ${copy_file_name} with file size: ${send_file_size}" # Set S3 connection configuration mcli alias set backup "${S3_ENDPOINT}" "${S3_ACCESS_KEY_ID}" "${S3_SECRET_ACCESS_KEY}" echo "Starting to copy ${copy_file_name} to ${info_copy_path}..." -# Create the bucket (Only enable if neccessary) -# mcli mb backup/${S3_BUCKET} +# Copying backup to S3 mcli cp "${copy_file_name}" backup/"${mcli_copy_path}" # Do nettoyage echo "Maid is here... Doing cleaning..." -rm -f "${POSTGRES_DB}.*" +rm --force "${POSTGRES_DB}".* # Do anounce -if [[ ${TELEGRAM_METHOD} == 'private' ]]; then - txt="Backed up ${copy_file_name} with file size: ${send_file_size} \ - to ${info_copy_path}" - hooks/private-webhook.sh "${txt}" -elif [[ ${TELEGRAM_METHOD} == 'external' ]]; then - txt="Backed up ${copy_file_name} with file size: ${send_file_size} \ - to ${info_copy_path}" - hooks/external-webhook.sh "${txt}" -else - echo "No notification methods selected" -fi +run-parts --reverse \ + --arg "${copy_file_name}" \ + --arg "${send_file_size}" \ + --arg "${info_copy_path}" \ + /hooks diff --git a/hooks/external-webhook.sh b/hooks/external-webhook.sh deleted file mode 100755 index 9b1d98c..0000000 --- a/hooks/external-webhook.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -# -# Use http request to send notification to Telegram chat through external Telegram API - -set -euo pipefail -IFS=$'\n\t' - -message_text="${1}" - -curl -s \ - --data "text=${message_text}" \ - --data "chat_id=${TELEGRAM_CHAT_ID}" \ - 'https://api.telegram.org/bot'${TELEGRAM_BOT_TOKEN}'/sendMessage' > /dev/null diff --git a/hooks/private-webhook.sh b/hooks/private-webhook.sh deleted file mode 100755 index 961ea16..0000000 --- a/hooks/private-webhook.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# -# Use http request to send notification to Telegram chat through local Telegram bot - -set -euo pipefail -IFS=$'\n\t' - -message_text="${1}" - -curl -XPOST \ - --url "${NOTIFICATION_URL}" \ - --header 'Content-Type: application/json' \ - --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${message_text}\"}" \ - --max-time 10 \ - --retry 5 diff --git a/hooks/send_private_notification b/hooks/send_private_notification new file mode 100755 index 0000000..ed603f5 --- /dev/null +++ b/hooks/send_private_notification @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat through local Telegram bot + +set -eo pipefail +IFS=$'\n\t' + +copy_file_name="${1}" +send_file_size="${2}" +info_copy_path="${3}" + +message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \ +to ${info_copy_path}" + +if [[ -n "${PRIVATE_NOTIFICATION_URL}" ]]; then + curl -XPOST \ + --url "${PRIVATE_NOTIFICATION_URL}" \ + --header 'Content-Type: application/json' \ + --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${message_text}\"}" \ + --max-time 10 \ + --retry 5 +fi diff --git a/hooks/send_telegram_message b/hooks/send_telegram_message new file mode 100755 index 0000000..cc3eb5e --- /dev/null +++ b/hooks/send_telegram_message @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat through external Telegram API + +set -eo pipefail +IFS=$'\n\t' + +copy_file_name="${1}" +send_file_size="${2}" +info_copy_path="${3}" + +message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \ +to ${info_copy_path}" + +if [[ -n "${TELEGRAM_BOT_TOKEN}" ]]; then + if [[ "${TELEGRAM_CHAT_ID}" =~ "^\d{5,32}" ]]; then + curl -s \ + --data "text=${message_text}" \ + --data "chat_id=${TELEGRAM_CHAT_ID}" \ + 'https://api.telegram.org/bot'${TELEGRAM_BOT_TOKEN}'/sendMessage' > /dev/null + else + echo "Telegram chatID doesn't matched to standard pattern. Probably \ + TELEGRAM_BOT_TOKEN variable was set by mistake" + fi +fi From 407dcf50da598b711f51a11228487f810d900711 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 15:10:52 +0200 Subject: [PATCH 20/26] DO-101 resolving discussion --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 022ff5f..2088b31 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,10 @@ docker run -d \ -e PRIVATE_NOTIFICATION_URL=http://webhook \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ - -e SCHEDULE=[Chosen_schedule][1] + -e SCHEDULE=[Chosen_schedule][^1] numdes/nd_postgres_backup:v*.*.* ``` -[1]: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*. +[^1]: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*. ## Variables ### `Gitlab Actions` *[variables](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository)*: @@ -79,7 +79,7 @@ docker run -d \ | POSTGRES_USER | - | Database user | | POSTGRES_PASSWORD | - | Database user password | | POSTGRES_EXTRA_OPTS | --blobs | Extra options `pg_dump` run | -| SCHEDULE | @daily | `go-cron` schedule. See *[this](https://github.com/siameseoriental/nd_postgres_backup/tree/DO-101#backup-using-go-cron)* | +| SCHEDULE | @daily | `go-cron` schedule. See [this](#backup-using-go-cron) | | HEALTHCHECK_PORT | 8080 | Port listening for cron-schedule health check. | | S3_ACCESS_KEY_ID | - | Key or username with RW access to bucket | | S3_SECRET_ACCESS_KEY | - | Secret or password for `S3_ACCESS_KEY_ID` | From cb9b89b61c0f8555de5f786d91d87e3311f46e4c Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 15:14:33 +0200 Subject: [PATCH 21/26] DO-101 resolving discussion --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2088b31..3c80079 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,10 @@ docker run -d \ -e PRIVATE_NOTIFICATION_URL=http://webhook \ -e TELEGRAM_CHAT_ID=point_to_notify_group \ -e POSTGRES_PORT=if_not_5432 \ - -e SCHEDULE=[Chosen_schedule][^1] + -e SCHEDULE=Chosen_schedule numdes/nd_postgres_backup:v*.*.* ``` -[^1]: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*. +:wave: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*. ## Variables ### `Gitlab Actions` *[variables](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository)*: From efdec350f67d0a9082d2f1366895f56181483661 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 16:39:37 +0200 Subject: [PATCH 22/26] DO-101 perform shellchek --- backup.sh | 2 +- hooks/send_telegram_message | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backup.sh b/backup.sh index 9cd4b4c..df9fdcf 100644 --- a/backup.sh +++ b/backup.sh @@ -38,7 +38,7 @@ tar --create \ "${POSTGRES_DB}.sql" # Count file size -send_file_size="$(ls -lh | grep "${copy_file_name}" | awk '{print $5}')" +send_file_size="$(ls -lh "${copy_file_name}" | awk '{print $5}')" echo "Created ${copy_file_name} with file size: ${send_file_size}" diff --git a/hooks/send_telegram_message b/hooks/send_telegram_message index cc3eb5e..3c56bc5 100755 --- a/hooks/send_telegram_message +++ b/hooks/send_telegram_message @@ -17,7 +17,7 @@ if [[ -n "${TELEGRAM_BOT_TOKEN}" ]]; then curl -s \ --data "text=${message_text}" \ --data "chat_id=${TELEGRAM_CHAT_ID}" \ - 'https://api.telegram.org/bot'${TELEGRAM_BOT_TOKEN}'/sendMessage' > /dev/null + 'https://api.telegram.org/bot'"${TELEGRAM_BOT_TOKEN}"'/sendMessage' > /dev/null else echo "Telegram chatID doesn't matched to standard pattern. Probably \ TELEGRAM_BOT_TOKEN variable was set by mistake" From 8f902270cbd5c1f393813c5d12795188c74dcd49 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 16:41:01 +0200 Subject: [PATCH 23/26] DO-101 remove verbose logging from backup.sh --- backup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backup.sh b/backup.sh index df9fdcf..1cf77f4 100644 --- a/backup.sh +++ b/backup.sh @@ -4,7 +4,7 @@ # or remote host. Created backup stores in S3 storage. On completion script calls # notification script hooks/00-webhook.sh which sends report to given Telegram Chat -set -euox pipefail +set -euo pipefail IFS=$'\n\t' # Will be name of directory in backet yyyy-mm-dd_HH:MM:SS From da1cd97d1f697afe73c0402152b0616ec338a549 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 16:43:50 +0200 Subject: [PATCH 24/26] DO-101 proofreading README.md --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 3c80079..9f23126 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,7 @@ docker run -d \ |TELEGRAM_BOT_TOKEN | Only used to call Telegram's public API | ### Environmental variables -| Name | Default | Description | -| | value | | +| Name | Default value | Description | |------------------------| :------ |------------------------------------------------| | POSTGRES_DB | - | Database name | | POSTGRES_HOST | - | PostgreSQL IP address or hostname | From ed6710b1f353cc5aef18387d69c44b0c65cb69d0 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 17:43:25 +0200 Subject: [PATCH 25/26] DO-101 fix mistake in Dockerfile and regexp --- Dockerfile | 2 +- hooks/send_telegram_message | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a82a8da..3aac76c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,7 +34,7 @@ ENV POSTGRES_DB="**None**" \ S3_ACCESS_KEY_ID="**None**" \ S3_SECRET_ACCESS_KEY="**None**" \ S3_BUCKET="**None**" \ - S3_ENDPOINT="**None**" \ + S3_ENDPOINT="**None**" COPY backup.sh /backup.sh RUN chmod +x backup.sh diff --git a/hooks/send_telegram_message b/hooks/send_telegram_message index 3c56bc5..b95e003 100755 --- a/hooks/send_telegram_message +++ b/hooks/send_telegram_message @@ -13,7 +13,7 @@ message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \ to ${info_copy_path}" if [[ -n "${TELEGRAM_BOT_TOKEN}" ]]; then - if [[ "${TELEGRAM_CHAT_ID}" =~ "^\d{5,32}" ]]; then + if [[ "${TELEGRAM_CHAT_ID}" =~ ^[0-9]{5,32}$ ]]; then curl -s \ --data "text=${message_text}" \ --data "chat_id=${TELEGRAM_CHAT_ID}" \ From dca60d1e4879ae81541bbcdcde0cbef38268b9d7 Mon Sep 17 00:00:00 2001 From: Petrov Yury Date: Mon, 31 Jul 2023 17:57:53 +0200 Subject: [PATCH 26/26] DO-101 proofread backup.sh --- backup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backup.sh b/backup.sh index 1cf77f4..02897b9 100644 --- a/backup.sh +++ b/backup.sh @@ -2,7 +2,8 @@ # # Script made for backup PostgreSQL database from local (${POSTGRES_HOST}=127.0.0.1) # or remote host. Created backup stores in S3 storage. On completion script calls -# notification script hooks/00-webhook.sh which sends report to given Telegram Chat +# notification scripts from hooks/ directory to send report to given Telegram Chat +# based on variables set private or public notification method will be selected set -euo pipefail IFS=$'\n\t' @@ -30,7 +31,6 @@ mcli_copy_path="${copy_path}/${copy_file_name}" info_copy_path="${S3_ENDPOINT}/${copy_path}" # Do compression -#tar -czvf "${copy_file_name}" "${POSTGRES_DB}.sql" tar --create \ --gzip \ --verbose \