diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..9b697f3 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,62 @@ +name: Docker Image CI + +on: + workflow_dispatch: + push: + branches: + - 'main' + - 'master' + tags: + - 'v*' + pull_request: + branches: + - 'main' + - 'master' + +env: + REGISTRY: docker.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + + build: + + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Workaround: https://github.com/docker/build-push-action/issues/461 + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + # Login against a Docker registry + # https://github.com/docker/login-action + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + # Extract metadata (tags, labels) for Docker + # https://github.com/docker/metadata-action + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + # Build and push Docker image with Buildx (don't push on PR) + # https://github.com/docker/build-push-action + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + with: + context: . + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3aac76c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +# syntax=docker/dockerfile:1.2 + +FROM debian:stable as base + +MAINTAINER NumDes + +LABEL org.opencontainers.image.vendor="Numerical Design LLC" +LABEL org.opencontainers.image.description="Docker image for universal postgres backups" + +ARG GOCRONVER=v0.0.10 + +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + curl \ + ca-certificates \ + postgresql-client \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && curl --fail --retry 4 --retry-all-errors -o /usr/local/bin/go-cron.gz -L https://github.com/prodrigestivill/go-cron/releases/download/$GOCRONVER/go-cron-linux-amd64.gz \ + && gzip -vnd /usr/local/bin/go-cron.gz && chmod a+x /usr/local/bin/go-cron + +RUN curl --location --output /usr/local/bin/mcli "https://dl.min.io/client/mc/release/linux-amd64/mc" && \ + chmod +x /usr/local/bin/mcli +RUN mcli -v + +ENV POSTGRES_DB="**None**" \ + POSTGRES_HOST="**None**" \ + POSTGRES_PORT=5432 \ + POSTGRES_USER="**None**" \ + POSTGRES_PASSWORD="**None**" \ + POSTGRES_EXTRA_OPTS="--blobs" \ + SCHEDULE="@daily" \ + HEALTHCHECK_PORT=8080 \ + S3_ACCESS_KEY_ID="**None**" \ + S3_SECRET_ACCESS_KEY="**None**" \ + S3_BUCKET="**None**" \ + S3_ENDPOINT="**None**" + +COPY backup.sh /backup.sh +RUN chmod +x backup.sh + +COPY hooks /hooks + +ENTRYPOINT ["/bin/sh", "-c"] +CMD ["exec /usr/local/bin/go-cron -s \"$SCHEDULE\" -p \"$HEALTHCHECK_PORT\" -- /backup.sh"] + +HEALTHCHECK --interval=5m --timeout=3s \ + CMD curl -f "http://localhost:$HEALTHCHECK_PORT/" || exit 1 diff --git a/README.md b/README.md index 0624fb5..9f23126 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,105 @@ Docker image for universal postgres backups # Roadmap -- [ ] Add support for S3 +- [X] Add support for S3 +- [X] Add CI/CD to publish image to DockerHub - [ ] Add retention policy settings by env vars -- [ ] Notify about backup status by HTTP-request \ No newline at end of file +- [X] Notify about backup status by HTTP-request +- [ ] Add docker-compose example + +## Docker build +```shell +docker build . -t numdes/nd_postgres_backup:v*.*.* +``` + +# Usage +## Backup manually: +```shell +docker run --rm -it \ + -e POSTGRES_HOST="FQDN-OR-IP" \ + -e POSTGRES_DB="DB-NAME" \ + -e POSTGRES_USER="DB-USER" \ + -e POSTGRES_PASSWORD="PASS" \ + -e S3_ENDPOINT=http://YOUR-S3 \ + -e S3_ACCESS_KEY_ID="KEY-ID" \ + -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ + -e S3_BUCKET="BUCKET-NAME" \ + -e PRIVATE_NOTIFICATION_URL=http://webhook \ + -e TELEGRAM_CHAT_ID=point_to_notify_group \ + -e POSTGRES_PORT=if_not_5432 \ + --entrypoint /bin/bash \ + numdes/nd_postgres_backup:v*.*.* +``` +To run backup, in active container shell call `backup.sh` script +```shell +./backup.sh +``` + +## Backup using `go-cron` +```shell +docker run -d \ + -e POSTGRES_HOST="FQDN-OR-IP" \ + -e POSTGRES_DB="DB-NAME" \ + -e POSTGRES_USER="DB-USER" \ + -e POSTGRES_PASSWORD="PASS" \ + -e S3_ENDPOINT=http://YOUR-S3 \ + -e S3_ACCESS_KEY_ID="KEY-ID" \ + -e S3_SECRET_ACCESS_KEY="KEY-SECRET" \ + -e S3_BUCKET="BUCKET-NAME" \ + -e PRIVATE_NOTIFICATION_URL=http://webhook \ + -e TELEGRAM_CHAT_ID=point_to_notify_group \ + -e POSTGRES_PORT=if_not_5432 \ + -e SCHEDULE=Chosen_schedule + numdes/nd_postgres_backup:v*.*.* +``` +:wave: By default `SCHEDULE` variable is set to `@daily` in case if you need other scheduling options, please refer to `go-cron` *[Documentation](https://pkg.go.dev/github.com/robfig/cron?utm_source=godoc#hdr-Predefined_schedules)*. + +## Variables +### `Gitlab Actions` *[variables](https://docs.github.com/en/actions/security-guides/encrypted-secrets#creating-encrypted-secrets-for-a-repository)*: +| Name | Description | +|-------------------|-----------------------------------------------------| +|DOCKERHUB_LOGIN | `Actions` Repository secret | +|DOCKERHUB_PASSWORD | `Actions` Repository secret | + +### Notification environmental variables +| Name | Description | +|---------------------------|-----------------------------------------------------| +|TELEGRAM_CHAT_ID | Notifying group | +|PRIVATE_NOTIFICATION_URL | Private notifier URL | +|TELEGRAM_BOT_TOKEN | Only used to call Telegram's public API | + +### Environmental variables +| Name | Default value | Description | +|------------------------| :------ |------------------------------------------------| +| POSTGRES_DB | - | Database name | +| POSTGRES_HOST | - | PostgreSQL IP address or hostname | +| POSTGRES_PORT | 5432 | Connection TCP port | +| POSTGRES_USER | - | Database user | +| POSTGRES_PASSWORD | - | Database user password | +| POSTGRES_EXTRA_OPTS | --blobs | Extra options `pg_dump` run | +| SCHEDULE | @daily | `go-cron` schedule. See [this](#backup-using-go-cron) | +| HEALTHCHECK_PORT | 8080 | Port listening for cron-schedule health check. | +| S3_ACCESS_KEY_ID | - | Key or username with RW access to bucket | +| S3_SECRET_ACCESS_KEY | - | Secret or password for `S3_ACCESS_KEY_ID` | +| S3_BUCKET | - | Name of bucket created for backups | +| S3_ENDPOINT | - | URL of S3 storage | + +### Notification selection + +It is possible to use either private Telegram bot if you have it or Telegram public API. + +In scenario with private bot `PRIVATE_NOTIFICATION_URL` must be set alongside with `TELEGRAM_CHAT_ID`. + +In scenario with Telegram's public API `TELEGRAM_BOT_TOKEN` must be set as it is received (`Use this token to access the HTTP API:`) from `@BotFather` Telegram Bot. Variable `TELEGRAM_CHAT_ID` must be a proper Telegram ID of bot + +In `docker ...` command need to replace: +``` + -e PRIVATE_NOTIFICATION_URL=http://webhook \ + -e TELEGRAM_CHAT_ID=point_to_notify_group \ +``` +to +``` + -e TELEGRAM_BOT_TOKEN='XXXXXXX:XXXXxxxxXXXXxxx' \ + -e TELEGRAM_CHAT_ID=000000000 \ +``` +- If `TELEGRAM_CHAT_ID` has a proper format (Only digits not less than 5 not more than 32) and `TELEGRAM_BOT_TOKEN` is set, script will try to send notification through Telegram's public API. diff --git a/backup.sh b/backup.sh new file mode 100644 index 0000000..02897b9 --- /dev/null +++ b/backup.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# +# Script made for backup PostgreSQL database from local (${POSTGRES_HOST}=127.0.0.1) +# or remote host. Created backup stores in S3 storage. On completion script calls +# notification scripts from hooks/ directory to send report to given Telegram Chat +# based on variables set private or public notification method will be selected + +set -euo pipefail +IFS=$'\n\t' + +# Will be name of directory in backet yyyy-mm-dd_HH:MM:SS +timestamp="$(date +%F_%T)" + +export PGPASSWORD=${POSTGRES_PASSWORD} + +# Will create base backup +echo "Creating backup of ${POSTGRES_DB} database. From ${POSTGRES_HOST} and port \ +is ${POSTGRES_PORT}. Username: ${POSTGRES_USER}. With following extra options: \ +${POSTGRES_EXTRA_OPTS}" +pg_dump --username="${POSTGRES_USER}" \ + --host="${POSTGRES_HOST}" \ + --port="${POSTGRES_PORT}" \ + --dbname="${POSTGRES_DB}" \ + "${POSTGRES_EXTRA_OPTS}" \ + > "${POSTGRES_DB}".sql + +# Declaring variables for informational purposes +copy_file_name="${POSTGRES_DB}.tar.gz" +copy_path="${S3_BUCKET}/${POSTGRES_DB}/${timestamp}" +mcli_copy_path="${copy_path}/${copy_file_name}" +info_copy_path="${S3_ENDPOINT}/${copy_path}" + +# Do compression +tar --create \ + --gzip \ + --verbose \ + --file "${copy_file_name}" \ + "${POSTGRES_DB}.sql" + +# Count file size +send_file_size="$(ls -lh "${copy_file_name}" | awk '{print $5}')" + +echo "Created ${copy_file_name} with file size: ${send_file_size}" + +# Set S3 connection configuration +mcli alias set backup "${S3_ENDPOINT}" "${S3_ACCESS_KEY_ID}" "${S3_SECRET_ACCESS_KEY}" + +echo "Starting to copy ${copy_file_name} to ${info_copy_path}..." + +# Copying backup to S3 +mcli cp "${copy_file_name}" backup/"${mcli_copy_path}" + +# Do nettoyage +echo "Maid is here... Doing cleaning..." +rm --force "${POSTGRES_DB}".* + +# Do anounce +run-parts --reverse \ + --arg "${copy_file_name}" \ + --arg "${send_file_size}" \ + --arg "${info_copy_path}" \ + /hooks diff --git a/hooks/send_private_notification b/hooks/send_private_notification new file mode 100755 index 0000000..ed603f5 --- /dev/null +++ b/hooks/send_private_notification @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat through local Telegram bot + +set -eo pipefail +IFS=$'\n\t' + +copy_file_name="${1}" +send_file_size="${2}" +info_copy_path="${3}" + +message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \ +to ${info_copy_path}" + +if [[ -n "${PRIVATE_NOTIFICATION_URL}" ]]; then + curl -XPOST \ + --url "${PRIVATE_NOTIFICATION_URL}" \ + --header 'Content-Type: application/json' \ + --data "{\"key\": \"${TELEGRAM_CHAT_ID}\", \"text\": \"${message_text}\"}" \ + --max-time 10 \ + --retry 5 +fi diff --git a/hooks/send_telegram_message b/hooks/send_telegram_message new file mode 100755 index 0000000..b95e003 --- /dev/null +++ b/hooks/send_telegram_message @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# Use http request to send notification to Telegram chat through external Telegram API + +set -eo pipefail +IFS=$'\n\t' + +copy_file_name="${1}" +send_file_size="${2}" +info_copy_path="${3}" + +message_text="Backed up ${copy_file_name} with file size: ${send_file_size} \ +to ${info_copy_path}" + +if [[ -n "${TELEGRAM_BOT_TOKEN}" ]]; then + if [[ "${TELEGRAM_CHAT_ID}" =~ ^[0-9]{5,32}$ ]]; then + curl -s \ + --data "text=${message_text}" \ + --data "chat_id=${TELEGRAM_CHAT_ID}" \ + 'https://api.telegram.org/bot'"${TELEGRAM_BOT_TOKEN}"'/sendMessage' > /dev/null + else + echo "Telegram chatID doesn't matched to standard pattern. Probably \ + TELEGRAM_BOT_TOKEN variable was set by mistake" + fi +fi