Permalink
Browse files

Merge pull request #813 from amazeeio/tweaks

Tweaks
  • Loading branch information...
Schnitzel committed Jan 3, 2019
2 parents e5931b2 + 882a31f commit 41744dbf0881a45531f976ce13e4f91497375245
Showing with 596 additions and 103 deletions.
  1. +1 −1 .lagoon.yml
  2. +10 −0 helpers/annotate-pvc-backup.sh
  3. +1 −0 images/mariadb-galera/Dockerfile
  4. +16 −0 images/mariadb-galera/fix-data-subpath.sh
  5. +1 −1 images/mariadb-galera/mysql-backup.sh
  6. +1 −1 images/mariadb/mysql-backup.sh
  7. +4 −7 images/oc-build-deploy-dind/build-deploy-docker-compose.sh
  8. +0 −11 images/oc-build-deploy-dind/openshift-templates/backup/schedule.yml
  9. +6 −1 images/oc-build-deploy-dind/openshift-templates/cli-persistent/deployment.yml
  10. +6 −1 images/oc-build-deploy-dind/openshift-templates/cli/deployment.yml
  11. +3 −0 images/oc-build-deploy-dind/openshift-templates/elasticsearch-cluster/statefulset.yml
  12. +3 −0 images/oc-build-deploy-dind/openshift-templates/elasticsearch/deployment.yml
  13. +2 −0 images/oc-build-deploy-dind/openshift-templates/elasticsearch/pvc.yml
  14. +3 −0 images/oc-build-deploy-dind/openshift-templates/mariadb-galera/deployment.yml
  15. +0 −1 images/oc-build-deploy-dind/openshift-templates/mariadb-galera/statefulset.yml
  16. +3 −0 images/oc-build-deploy-dind/openshift-templates/mariadb/deployment.yml
  17. +2 −0 images/oc-build-deploy-dind/openshift-templates/mariadb/pvc.yml
  18. +7 −0 images/oc-build-deploy-dind/openshift-templates/nginx-php-persistent/deployment.yml
  19. +1 −1 images/oc-build-deploy-dind/openshift-templates/nginx-php-persistent/pvc.yml
  20. +210 −0 images/oc-build-deploy-dind/openshift-templates/nginx-php-redis-persistent/deployment.yml
  21. +59 −0 images/oc-build-deploy-dind/openshift-templates/nginx-php-redis-persistent/pvc.yml
  22. +59 −0 images/oc-build-deploy-dind/openshift-templates/nginx-php-redis-persistent/routes.yml
  23. +70 −0 images/oc-build-deploy-dind/openshift-templates/nginx-php-redis-persistent/services.yml
  24. +4 −0 images/oc-build-deploy-dind/openshift-templates/nginx-php/deployment.yml
  25. +4 −0 images/oc-build-deploy-dind/openshift-templates/nginx/deployment.yml
  26. +7 −0 images/oc-build-deploy-dind/openshift-templates/node-persistent/deployment.yml
  27. +1 −1 images/oc-build-deploy-dind/openshift-templates/node-persistent/pvc.yml
  28. +4 −0 images/oc-build-deploy-dind/openshift-templates/node/deployment.yml
  29. +3 −0 images/oc-build-deploy-dind/openshift-templates/redis-persistent/deployment.yml
  30. +2 −0 images/oc-build-deploy-dind/openshift-templates/redis-persistent/pvc.yml
  31. +3 −0 images/oc-build-deploy-dind/openshift-templates/solr/deployment.yml
  32. +1 −1 images/oc-build-deploy-dind/openshift-templates/solr/pvc.yml
  33. +4 −0 images/oc-build-deploy-dind/openshift-templates/varnish/deployment.yml
  34. +1 −0 images/oc-build-deploy-dind/scripts/git-checkout-pull-merge.sh
  35. +1 −0 images/oc-build-deploy-dind/scripts/git-checkout-pull.sh
  36. +1 −0 node-packages/commons/src/api.js
  37. +3 −1 services/api-db/Dockerfile
  38. +12 −2 services/logs-collector/.lagoon.yml
  39. +0 −1 services/logs-db-curator/Dockerfile
  40. +34 −30 services/logs-db-curator/actions.yml
  41. +1 −1 services/logs-db-curator/curator.yml
  42. +10 −1 services/logs-forwarder/.lagoon.multi.yml
  43. +10 −1 services/logs-forwarder/.lagoon.single.yml
  44. +1 −1 services/logs2logs-db/pipeline/router-application-logs.conf
  45. +1 −1 services/storage-calculator/calculate-storage.sh
  46. +20 −37 services/webhooks2tasks/src/handlers/resticbackupSnapshotFinished.js
@@ -56,5 +56,5 @@ environments:
service: auto-idler
- name: curator
schedule: '01 0 * * * '
command: /usr/bin/curator --config curator.yml actions.yml
command: /usr/bin/curator --config /curator/curator.yml /curator/actions.yml
service: logs-db-curator
@@ -0,0 +1,10 @@
#!/bin/bash

##
## This annotates all PVCs with the name `solr` and `nginx` with appuio.ch/backup="true" in order that the restic backup system will back them up
##

oc get pvc --all-namespaces | grep solr | sed '1d' | awk '{ print $2, "--namespace", $1 }' | while read line; do oc annotate --overwrite pvc $line appuio.ch/backup="true"; done
oc get pvc --all-namespaces | grep nginx | sed '1d' | awk '{ print $2, "--namespace", $1 }' | while read line; do oc annotate --overwrite pvc $line appuio.ch/backup="true"; done

oc get --all-namespaces pod -l 'service in (cli)' | sed '1d' | awk '{ print "--namespace", $1, $2 }' | while read line; do oc annotate --overwrite pod $line appuio.ch/backupcommand='/bin/bash -c "if [[ $MARIADB_HOST ]]; then mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"'; done
@@ -92,6 +92,7 @@ RUN apk add --no-cache --virtual .common-run-deps \
rm -rf /var/cache/apk/*

COPY docker-entrypoint.bash /lagoon/entrypoints/9999-mariadb-entrypoint
COPY fix-data-subpath.sh /lagoon/entrypoints/100-fix-data-subpath
COPY maxscale-entrypoint.sh /lagoon/entrypoints/80-maxscale
COPY maxscale.sql /docker-entrypoint-initdb.d/maxscale.sql
COPY maxscale.cnf /etc/maxscale.cnf
@@ -0,0 +1,16 @@
#!/usr/bin/env bash

# previously mariadb-galera used a `subPath` volume mount, which caused the folder `data` from a PVC to be mounted into `/var/lib/mysql`
# somehow this system broke on some Kubernetes versions so we mount the PVC directly into `/var/lib/mysql`
# this script will move the files from within /var/lib/mysql/data to /var/lib/mysql But only if this never happened before.

if [ -z "$(ls -A /var/lib/mysql | grep -v data)" ]; then
# /var/lib/mysql has only a `data` folder, is therefore not migrated yet
if [ ! -z "$(ls -A /var/lib/mysql/data)" ]; then
# /var/lib/mysql/data is not empty so we move all files from within /var/lib/mysql/data into /var/lib/mysql
echo "mysql files found in old location /var/lib/mysql/data, moving them into /var/lib/mysql"
mv /var/lib/mysql/data/* /var/lib/mysql
# ignore errors for dot files (as a non existing hidden dot file cause an error that we don't need)
mv /var/lib/mysql/data/{.[!.]}* /var/lib/mysql || true
fi
fi
@@ -78,7 +78,7 @@ function backup_database(){
backup_file="$BACKUP_DIR/$TIMESTAMP.$database.sql.gz"
output="${output}${database} => $backup_file\n"
echo_status "...backing up $count of $total databases: $database"
$(mysqldump $(mysql_login) $database | gzip -9 > $backup_file)
$(mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction $(mysql_login) $database | gzip -9 > $backup_file)
}

function backup_databases(){
@@ -78,7 +78,7 @@ function backup_database(){
backup_file="$BACKUP_DIR/$TIMESTAMP.$database.sql.gz"
output="${output}${database} => $backup_file\n"
echo_status "...backing up $count of $total databases: $database"
$(mysqldump $(mysql_login) $database | gzip -9 > $backup_file)
$(mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction $(mysql_login) $database | gzip -9 > $backup_file)
}

function backup_databases(){
@@ -45,7 +45,7 @@ do
SERVICE_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.type custom)

# Allow the servicetype to be overriden by environment in .lagoon.yml
ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.types.$SERVICE_NAME false)
ENVIRONMENT_SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.types.$SERVICE_NAME false)
if [ ! $ENVIRONMENT_SERVICE_TYPE_OVERRIDE == "false" ]; then
SERVICE_TYPE=$ENVIRONMENT_SERVICE_TYPE_OVERRIDE
fi
@@ -242,7 +242,7 @@ do
SERVICE_NAME=${SERVICE_TYPES_ENTRY_SPLIT[0]}
SERVICE_TYPE=${SERVICE_TYPES_ENTRY_SPLIT[1]}

SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.types.$SERVICE_NAME false)
SERVICE_TYPE_OVERRIDE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.types.$SERVICE_NAME false)
if [ ! $SERVICE_TYPE_OVERRIDE == "false" ]; then
SERVICE_TYPE=$SERVICE_TYPE_OVERRIDE
fi
@@ -369,9 +369,6 @@ if oc get --insecure-skip-tls-verify customresourcedefinition schedules.backup.a
BACKUP_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "H 0 * * *")
TEMPLATE_PARAMETERS+=(-p BACKUP_SCHEDULE="${BACKUP_SCHEDULE}")

PRUNE_SCHEDULE=$( /oc-build-deploy/scripts/convert-crontab.sh "${OPENSHIFT_PROJECT}" "H 3 * * *")
TEMPLATE_PARAMETERS+=(-p PRUNE_SCHEDULE="${PRUNE_SCHEDULE}")

OPENSHIFT_TEMPLATE="/oc-build-deploy/openshift-templates/backup/schedule.yml"
. /oc-build-deploy/scripts/exec-openshift-resources.sh
fi
@@ -669,7 +666,7 @@ do
fi

OVERRIDE_TEMPLATE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.$COMPOSE_SERVICE.labels.lagoon\\.template false)
ENVIRONMENT_OVERRIDE_TEMPLATE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.templates.$SERVICE_NAME false)
ENVIRONMENT_OVERRIDE_TEMPLATE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.templates.$SERVICE_NAME false)
if [[ "${OVERRIDE_TEMPLATE}" == "false" && "${ENVIRONMENT_OVERRIDE_TEMPLATE}" == "false" ]]; then # No custom template defined in docker-compose or .lagoon.yml, using the given service ones
# Generate deployment if service type defines it
OPENSHIFT_DEPLOYMENT_TEMPLATE="/oc-build-deploy/openshift-templates/${SERVICE_TYPE}/deployment.yml"
@@ -733,7 +730,7 @@ do
SERVICE_ROLLOUT_TYPE=$(cat $DOCKER_COMPOSE_YAML | shyaml get-value services.${SERVICE_NAME}.labels.lagoon\\.rollout deploymentconfigs)

# Allow the rollout type to be overriden by environment in .lagoon.yml
ENVIRONMENT_SERVICE_ROLLOUT_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH}.rollouts.${SERVICE_NAME} false)
ENVIRONMENT_SERVICE_ROLLOUT_TYPE=$(cat .lagoon.yml | shyaml get-value environments.${BRANCH//./\\.}.rollouts.${SERVICE_NAME} false)
if [ ! $ENVIRONMENT_SERVICE_ROLLOUT_TYPE == "false" ]; then
SERVICE_ROLLOUT_TYPE=$ENVIRONMENT_SERVICE_ROLLOUT_TYPE
fi
@@ -34,22 +34,11 @@ parameters:
- name: BACKUP_SCHEDULE
description: Schedule of the Backup in Cron format
required: true
- name: PRUNE_SCHEDULE
description: Schedule of the Pruning in Cron format
required: true
objects:
- apiVersion: backup.appuio.ch/v1alpha1
kind: Schedule
metadata:
name: backup-schedule
spec:
backup:
keepJobs: 2
schedule: '${BACKUP_SCHEDULE}'
check:
schedule: 0 0 * * 0
prune:
retention:
keepDaily: 7
keepWeekly: 4
schedule: '${PRUNE_SCHEDULE}'
@@ -73,7 +73,8 @@ objects:
template:
metadata:
annotations:
appuio.ch/backupcommand: mysqldump --all-databases -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD
appuio.ch/backupcommand: /bin/sh -c "if [[ $MARIADB_HOST ]]; then mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"
backup.appuio.ch/file-extension: .mysql.sql
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
@@ -85,6 +86,10 @@ objects:
key: autoscaled
operator: Equal
value: 'true'
- effect: NoSchedule
key: appuio.ch/autoscaled
operator: Equal
value: 'true'
volumes:
- name: ${PERSISTENT_STORAGE_NAME}
persistentVolumeClaim:
@@ -59,7 +59,8 @@ objects:
template:
metadata:
annotations:
appuio.ch/backupcommand: mysqldump --all-databases -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD
appuio.ch/backupcommand: /bin/sh -c "if [[ $MARIADB_HOST ]]; then mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction --no-create-db -h $MARIADB_HOST -u $MARIADB_USERNAME -p$MARIADB_PASSWORD $MARIADB_DATABASE; fi"
backup.appuio.ch/file-extension: .mysql.sql
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
@@ -71,6 +72,10 @@ objects:
key: autoscaled
operator: Equal
value: 'true'
- effect: NoSchedule
key: appuio.ch/autoscaled
operator: Equal
value: 'true'
volumes:
- name: lagoon-sshkey
secret:
@@ -50,6 +50,9 @@ objects:
service: ${SERVICE_NAME}
template:
metadata:
annotations:
appuio.ch/backupcommand: /bin/sh -c "/bin/busybox tar -cf - -C /usr/share/elasticsearch/data ."
backup.appuio.ch/file-extension: .${SERVICE_NAME}.tar
labels:
branch: ${SAFE_BRANCH}
project: ${SAFE_PROJECT}
@@ -61,6 +61,9 @@ objects:
type: ${DEPLOYMENT_STRATEGY}
template:
metadata:
annotations:
appuio.ch/backupcommand: /bin/sh -c "/bin/busybox tar -cf - -C /usr/share/elasticsearch/data ."
backup.appuio.ch/file-extension: .${SERVICE_NAME}.tar
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
@@ -41,6 +41,8 @@ objects:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
appuio.ch/backup: "false"
name: ${SERVICE_NAME}
spec:
accessModes:
@@ -58,6 +58,9 @@ objects:
type: ${DEPLOYMENT_STRATEGY}
template:
metadata:
annotations:
appuio.ch/backupcommand: /bin/sh -c "mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction --all-databases"
backup.appuio.ch/file-extension: .${SERVICE_NAME}.sql
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
@@ -90,7 +90,6 @@ objects:
volumeMounts:
- name: ${SERVICE_NAME}-galera
mountPath: /var/lib/mysql
subPath: data
env:
- name: POD_NAMESPACE
valueFrom:
@@ -61,6 +61,9 @@ objects:
type: ${DEPLOYMENT_STRATEGY}
template:
metadata:
annotations:
appuio.ch/backupcommand: /bin/sh -c "mysqldump --events --routines --quick --add-locks --no-autocommit --single-transaction --all-databases"
backup.appuio.ch/file-extension: .${SERVICE_NAME}.sql
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
@@ -41,6 +41,8 @@ objects:
- apiVersion: v1
kind: PersistentVolumeClaim
metadata:
annotations:
appuio.ch/backup: "false"
name: ${SERVICE_NAME}-data
spec:
accessModes:
@@ -76,6 +76,9 @@ objects:
type: ${DEPLOYMENT_STRATEGY}
template:
metadata:
annotations:
appuio.ch/backupcommand: /bin/sh -c "/bin/busybox tar -cf - -C ${PERSISTENT_STORAGE_PATH} ."
backup.appuio.ch/file-extension: .${SERVICE_NAME}.tar
creationTimestamp: null
labels:
service: ${SERVICE_NAME}
@@ -87,6 +90,10 @@ objects:
key: autoscaled
operator: Equal
value: 'true'
- effect: NoSchedule
key: appuio.ch/autoscaled
operator: Equal
value: 'true'
volumes:
- name: ${SERVICE_NAME}
persistentVolumeClaim:
@@ -48,7 +48,7 @@ objects:
kind: PersistentVolumeClaim
metadata:
annotations:
appuio.ch/backup: "true"
appuio.ch/backup: "false"
name: ${SERVICE_NAME}
spec:
accessModes:
Oops, something went wrong.

0 comments on commit 41744db

Please sign in to comment.