forked from blacklabelops/volumerize
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Reorganise and improve database images
- specify the database to backup (only mysql) - specify the source variable in which to store the database dumps - use env check globally - add support for docker secrets (see #9, includes database passwords, gpg private key and passphrase) - remove mysql restore script since it does not work and is obsolete because of restore command see #5
- Loading branch information
Showing
15 changed files
with
193 additions
and
95 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| version: "3" | ||
|
|
||
| services: | ||
| mongodb: | ||
| image: mongo | ||
| ports: | ||
| - 27017:27017 | ||
| environment: | ||
| - MONGO_INITDB_ROOT_USERNAME=root | ||
| - MONGO_INITDB_ROOT_PASSWORD=1234 | ||
| volumes: | ||
| - mongodb:/data/db | ||
|
|
||
| volumerize: | ||
| image: fekide/volumerize:mongodb | ||
| environment: | ||
| - VOLUMERIZE_SOURCE=/source | ||
| - VOLUMERIZE_TARGET=file:///backup | ||
| - MONGO_USERNAME=root | ||
| - MONGO_PASSWORD=1234 | ||
| - MONGO_PORT=27017 | ||
| - MONGO_HOST=mongodb | ||
| volumes: | ||
| - volumerize-cache:/volumerize-cache | ||
| - backup:/backup | ||
| depends_on: | ||
| - mongodb | ||
|
|
||
| volumes: | ||
| volumerize-cache: | ||
| mongodb: | ||
| backup: |
16 changes: 13 additions & 3 deletions
16
prepost_strategies/mongodb/postexecute/restore/mongorestore.sh
100644 → 100755
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,17 @@ | ||
| source /preexecute/utils/check-env.sh | ||
| #!/bin/bash | ||
|
|
||
| check_env "mongorestore" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" | ||
| set -o errexit | ||
|
|
||
| source /opt/volumerize/env.sh | ||
|
|
||
| VOLUMERIZE_MONGO_SOURCE=${VOLUMERIZE_MONGO_SOURCE:-VOLUMERIZE_SOURCE} | ||
| export MONGO_SOURCE=${!VOLUMERIZE_MONGO_SOURCE} | ||
|
|
||
| file_env "MONGO_PASSWORD" | ||
| check_env "mongorestore" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" "MONGO_SOURCE" | ||
|
|
||
| MONGO_SOURCE=${MONGO_SOURCE}/volumerize-mongo | ||
|
|
||
| echo "mongorestore starts" | ||
| mongorestore --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" ${VOLUMERIZE_SOURCE} | ||
| mongorestore --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" ${MONGO_SOURCE} | ||
| echo "Import done" |
20 changes: 15 additions & 5 deletions
20
prepost_strategies/mongodb/preexecute/backup/mongodump.sh
100644 → 100755
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,9 +1,19 @@ | ||
| source /preexecute/utils/check-env.sh | ||
| #!/bin/bash | ||
|
|
||
| check_env "mongodump" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" | ||
| set -o errexit | ||
|
|
||
| echo "Creating $VOLUMERIZE_SOURCE folder if not exists" | ||
| mkdir -p $VOLUMERIZE_SOURCE | ||
| source /opt/volumerize/env.sh | ||
|
|
||
| VOLUMERIZE_MONGO_SOURCE=${VOLUMERIZE_MONGO_SOURCE:-VOLUMERIZE_SOURCE} | ||
| export MONGO_SOURCE=${!VOLUMERIZE_MONGO_SOURCE} | ||
|
|
||
| file_env "MONGO_PASSWORD" | ||
| check_env "mongodump" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" "MONGO_SOURCE" | ||
|
|
||
| MONGO_SOURCE=${MONGO_SOURCE}/volumerize-mongo | ||
|
|
||
| echo "Creating $MONGO_SOURCE folder if not exists" | ||
| mkdir -p $MONGO_SOURCE | ||
|
|
||
| echo "mongodump starts" | ||
| mongodump --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" --out ${VOLUMERIZE_SOURCE} | ||
| mongodump --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" --out ${MONGO_SOURCE} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -7,4 +7,3 @@ RUN apk add --no-cache \ | |
|
|
||
| COPY postexecute /postexecute | ||
| COPY preexecute /preexecute | ||
| COPY scripts /etc/volumerize | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,34 @@ | ||
| version: "3" | ||
|
|
||
| services: | ||
| mariadb: | ||
| image: mariadb | ||
| ports: | ||
| - 3306:3306 | ||
| environment: | ||
| - MYSQL_ROOT_PASSWORD=1234 | ||
| - MYSQL_DATABASE=somedatabase | ||
| volumes: | ||
| - mariadb:/var/lib/mysql | ||
|
|
||
| volumerize: | ||
| image: fekide/volumerize:mysql | ||
| environment: | ||
| - VOLUMERIZE_SOURCE1=/source | ||
| - VOLUMERIZE_TARGET1=file:///backup | ||
| - MYSQL_USERNAME=root | ||
| - MYSQL_PASSWORD=1234 | ||
| - MYSQL_HOST=mariadb | ||
| - MYSQL_DATABASE=somedatabase | ||
| - VOLUMERIZE_MYSQL_SOURCE=VOLUMERIZE_SOURCE1 | ||
| - DEBUG=true | ||
| volumes: | ||
| - volumerize-cache:/volumerize-cache | ||
| - backup:/backup | ||
| depends_on: | ||
| - mariadb | ||
|
|
||
| volumes: | ||
| volumerize-cache: | ||
| mariadb: | ||
| backup: |
12 changes: 10 additions & 2 deletions
12
prepost_strategies/mysql/postexecute/restore/mysqlimport.sh
100644 → 100755
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,15 @@ | ||
| source /preexecute/utils/check-env.sh | ||
| #!/bin/bash | ||
|
|
||
| set -o errexit | ||
|
|
||
| source /opt/volumerize/env.sh | ||
|
|
||
| VOLUMERIZE_MYSQL_SOURCE=${VOLUMERIZE_MYSQL_SOURCE:-VOLUMERIZE_SOURCE} | ||
| export MYSQL_SOURCE=${!VOLUMERIZE_MYSQL_SOURCE} | ||
|
|
||
| file_env "MYSQL_PASSWORD" | ||
| check_env "mysqlimport" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" | ||
|
|
||
| echo "mysql import starts" | ||
| pv ${VOLUMERIZE_SOURCE}/volumerize-mysql/dump-${MYSQL_DATABASE}.sql | mysql -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE | ||
| pv ${MYSQL_SOURCE}/volumerize-mysql/dump-${MYSQL_DATABASE}.sql | mysql -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE | ||
| echo "Import done" |
31 changes: 18 additions & 13 deletions
31
prepost_strategies/mysql/preexecute/backup/mysqldump.sh
100644 → 100755
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,19 +1,24 @@ | ||
| source /preexecute/utils/check-env.sh | ||
| #!/bin/bash | ||
|
|
||
| check_env "Mysqldump" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" | ||
| set -o errexit | ||
|
|
||
| echo "Creating $VOLUMERIZE_SOURCE folder if not exists" | ||
| mkdir -p $VOLUMERIZE_SOURCE/volumerize-mysql/ | ||
| source /opt/volumerize/env.sh | ||
|
|
||
| log "Starting automatic repair and optimize for all databases..." | ||
| mysqlcheck -h ${MYSQL_HOST} -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} --all-databases --optimize --auto-repair --silent 2>&1 | ||
| VOLUMERIZE_MYSQL_SOURCE=${VOLUMERIZE_MYSQL_SOURCE:-VOLUMERIZE_SOURCE} | ||
| export MYSQL_SOURCE=${!VOLUMERIZE_MYSQL_SOURCE} | ||
|
|
||
| file_env "MYSQL_PASSWORD" | ||
| check_env "Mysqldump" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_SOURCE" "MYSQL_DATABASE" | ||
|
|
||
| for MYSQL_DATABASE in `mysql -h ${MYSQL_HOST} -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} -Bse 'show databases'`; do | ||
| echo "Creating ${MYSQL_SOURCE}/volumerize-mysql folder if not exists" | ||
| mkdir -p ${MYSQL_SOURCE}/volumerize-mysql | ||
|
|
||
| echo "Starting automatic repair and optimize for all databases..." | ||
| mysqlcheck -h ${MYSQL_HOST} -u${MYSQL_USERNAME} -p${MYSQL_PASSWORD} --all-databases --optimize --auto-repair --silent 2>&1 | ||
|
|
||
| # Based on this answer https://stackoverflow.com/a/32361604 | ||
| SIZE_BYTES=$(mysql --skip-column-names -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} ${MYSQL_DATABASE} -e "SELECT ROUND(SUM(data_length * 0.8), 0) FROM information_schema.TABLES WHERE table_schema='${MYSQL_DATABASE}';") | ||
| # Based on this answer https://stackoverflow.com/a/32361604 | ||
| SIZE_BYTES=$(mysql --skip-column-names -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} ${MYSQL_DATABASE} -e "SELECT ROUND(SUM(data_length * 0.8), 0) FROM information_schema.TABLES WHERE table_schema='${MYSQL_DATABASE}';") | ||
| [[ ${SIZE_BYTES} == NULL ]] && SIZE_BYTES=0 | ||
|
|
||
| echo "mysqldump starts (Progress is aproximated)" | ||
| mysqldump --databases "${MYSQL_DATABASE}" --single-transaction --add-drop-database --user="${MYSQL_USERNAME}" --password="${MYSQL_PASSWORD}" --host="${MYSQL_HOST}" | pv --progress --size "$SIZE_BYTES" > ${VOLUMERIZE_SOURCE}/volumerize-mysql/dump-${MYSQL_DATABASE}.sql | ||
|
|
||
| done | ||
| echo "mysqldump starts for database ${MYSQL_DATABASE} (Progress is aproximated)" | ||
| mysqldump --single-transaction --add-drop-database --user="${MYSQL_USERNAME}" --password="${MYSQL_PASSWORD}" --host="${MYSQL_HOST}" --databases "${MYSQL_DATABASE}" | pv --progress --size "${SIZE_BYTES:-0}" > ${MYSQL_SOURCE}/volumerize-mysql/dump-${MYSQL_DATABASE}.sql |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.