feat: add backups plugin

drydock_backups/Dockerfile

@@ -0,0 +1,21 @@
+FROM mysql:5.7-debian as base
+
+RUN apt-get update && apt-get install -y wget gnupg2
+
+RUN wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | apt-key add -
+
+RUN echo "deb http://repo.mongodb.org/apt/debian buster/mongodb-org/6.0 main" | tee /etc/apt/sources.list.d/mongodb-org-6.0.list
+
+RUN apt-get update && apt-get install -y \
+    mongodb-org \
+    awscli \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -m backupuser
+RUN echo "backupuser ALL=(ALL) NOPASSWD: /usr/bin/mysql, /usr/bin/mongodump" >> /etc/sudoers
+
+USER backupuser
+
+WORKDIR /home/backupuser
+
+RUN echo

drydock_backups/README.rst

@@ -0,0 +1,51 @@
+Drydock-backuyps
+================
+
+This is a tutor plugin used to easily store backups of mysql and mongodb databases through k8s cronjobs. It backups the databases and stores them in a bucket.
+
+This plugin assumes that the destination bucket is already created and that the credentials to access it are already configured (Works with S3 and Minio buckets indicating his endpoint url through `BACKUP_CUSTOM_STORAGE_ENDPOINT`).
+
+Installation and activation
+---------------------------
+
+It is included with the Drydock installation, so there's no need to install it separately.
+
+You can enable it adding `drydock-backups` to the `plugins` section of the `config.yml` file.
+
+Configuration variables
+-----------------------
+
+- **BACKUP_IMAGE**: The image used to run the cronjob. (default: `ednxops/shipyard-utils:v1.0.0`)
+- **BACKUP_CRON_SCHEDULE**: Cron schedule to run the backup. (default: `0 2 * * *`)
+- **BACKUP_AWS_ACCESS_KEY**: AWS access key to access the bucket or minIO user.
+- **BACKUP_AWS_SECRET_KEY**: AWS secret key to access the bucket or minIO password.
+- **BACKUP_BUCKET_NAME**: Name of the bucket where the backups will be stored.
+- **BACKUP_BUCKET_PATH**: Path inside the bucket where the backups will be stored.
+- **BACKUP_CUSTOM_STORAGE_ENDPOINT**: Custom endpoint to access the bucket. (default: `None`)
+- **BACKUP_K8S_USE_EPHEMERAL_VOLUMES**: Use ephemeral volumes to set up the cronjob. (default: `False`)
+- **BACKUP_K8S_EPHEMERAL_VOLUME_SIZE**: Size of the ephemeral volume. (default: `8Gi`)
+- **BACKUP_MYSQL_USERNAME**: Username to access the mysql database. (default: `{{ MYSQL_ROOT_USERNAME }}`)
+- **BACKUP_MYSQL_PASSWORD**: Password to access the mysql database. (default: `{{ MYSQL_ROOT_PASSWORD }}`)
+- **BACKUP_MONGO_PASSWORD**: Password to access the mongodb database. (default: `{{ MONGODB_PASSWORD }}`)
+- **BACKUP_MONGO_USERNAME**: Username to access the mongodb database. (default: `{{ MONGODB_USERNAME }}`)
+
+You can set ups these variables in the `config.yml` file.
+
+Docker image
+------------
+
+The docker image used to run the cronjob is `ednxops/shipyard-utils:v1.0.0` and it is available in `DockerHub <https://hub.docker.com/r/ednxops/shipyard-utils>`_.
+
+The dockerfile used to build the image is the Dockerfile located in the drydock-backups folder of this repository. It is built manually and pushed to DockerHub with the following commands:
+
+.. code-block:: bash
+
+    docker build -t ednxops/shipyard-utils:{{ TAG }} drydock-backups/
+    docker push ednxops/shipyard-utils:{{ TAG }}
+
+Utilities
+---------
+
+In the path `/utils` we have some utilities to help us managing the backups.
+
+- **s3_backups_expiration**: This terraform script is to add to S3 bucket lifecycle rules. Is used to delete the backups that are older than a certain number of days.

drydock_backups/__about__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

drydock_backups/patches/drydock-multipurpose-jobs

@@ -0,0 +1,131 @@
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: backup-mysql-cron
+  labels:
+    app.kubernetes.io/component: cronjob
+spec:
+  suspend: false
+  schedule: {{ BACKUP_CRON_SCHEDULE }}
+  startingDeadlineSeconds: 900
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: backup
+              image: {{ BACKUP_IMAGE }}
+              env:
+                - name: MYSQL_HOST
+                  value: '{{ MYSQL_HOST }}'
+                - name: MYSQL_PORT
+                  value: '{{ MYSQL_PORT }}'
+                - name: MYSQL_ROOT_USERNAME
+                  value: '{{ BACKUP_MYSQL_USERNAME }}'
+                - name: MYSQL_ROOT_PASSWORD
+                  value: '{{ BACKUP_MYSQL_PASSWORD }}'
+                - name: AWS_ACCESS_KEY_ID
+                  value: '{{ BACKUP_AWS_ACCESS_KEY }}'
+                - name: AWS_SECRET_ACCESS_KEY
+                  value: '{{ BACKUP_AWS_SECRET_KEY }}'
+                - name: S3_BUCKET_NAME
+                  value: '{{ BACKUP_BUCKET_NAME }}'
+                - name: BUCKET_PATH
+                  value: '{{ BACKUP_BUCKET_PATH }}'
+                {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %}
+                - name: BACKUP_CUSTOM_STORAGE_ENDPOINT
+                  value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}'
+                {% endif %}
+              {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+              volumeMounts:
+                - mountPath: /data/
+                  name: backup-volume
+              {% endif %}
+              command: ["/bin/sh", "-c"]
+              args: ["FILENAME=$(date +'%Y-%m-%d').sql.gz && \
+                    mysqldump --column-statistics=0 -u $MYSQL_ROOT_USERNAME -h $MYSQL_HOST -P $MYSQL_PORT \
+                    --password=$MYSQL_ROOT_PASSWORD --all-databases --single-transaction --flush-logs \
+                    | gzip > $FILENAME && aws {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %} --endpoint-url $BACKUP_CUSTOM_STORAGE_ENDPOINT {% endif %} s3 mv $FILENAME s3://$S3_BUCKET_NAME/$BUCKET_PATH/mysql/"]
+          {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+          volumes:
+            - name: backup-volume
+              ephemeral:
+                volumeClaimTemplate:
+                  spec:
+                    accessModes:
+                      - ReadWriteOnce
+                    resources:
+                      requests:
+                        storage: {{ BACKUP_K8S_EPHEMERAL_VOLUME_SIZE }}
+          {% endif %}
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: backup-mongo-cron
+  labels:
+    app.kubernetes.io/component: cronjob
+spec:
+  suspend: false
+  schedule: {{ BACKUP_CRON_SCHEDULE }}
+  startingDeadlineSeconds: 900
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 3
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          restartPolicy: OnFailure
+          containers:
+            - name: backup
+              image: {{ BACKUP_IMAGE }}
+              env:
+                - name: MONGODB_HOST
+                  value: '{{ MONGODB_HOST }}'
+                - name: MONGODB_PORT
+                  value: '{{ MONGODB_PORT }}'
+                - name: MONGODB_USERNAME
+                  value: '{{ BACKUP_MONGO_USERNAME }}'
+                - name: MONGODB_PASSWORD
+                  value: '{{ BACKUP_MONGO_PASSWORD }}'
+                - name: MONGODB_DATABASES
+                  value: '{{ MONGODB_DATABASE }}'
+                - name: AWS_ACCESS_KEY_ID
+                  value: '{{ BACKUP_AWS_ACCESS_KEY }}'
+                - name: AWS_SECRET_ACCESS_KEY
+                  value: '{{ BACKUP_AWS_SECRET_KEY }}'
+                - name: S3_BUCKET_NAME
+                  value: '{{ BACKUP_BUCKET_NAME }}'
+                - name: BUCKET_PATH
+                  value: '{{ BACKUP_BUCKET_PATH }}'
+                  {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %}
+                - name: BACKUP_CUSTOM_STORAGE_ENDPOINT
+                  value: '{{ BACKUP_CUSTOM_STORAGE_ENDPOINT }}'
+                  {% endif %}
+              {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+              volumeMounts:
+                - mountPath: /data/
+                  name: backup-volume
+              {% endif %}
+              command: ["/bin/sh", "-c"]
+              args: ["FILENAME=$(date +'%Y-%m-%d').gz && \
+              mongodump {% if MONGODB_USERNAME %} --username $MONGODB_USERNAME --password $MONGODB_PASSWORD --authenticationDatabase=admin {% endif %}\
+              --host $MONGODB_HOST:$MONGODB_PORT \
+              -d $MONGODB_DATABASES --gzip \
+              --archive=$FILENAME && aws {% if BACKUP_CUSTOM_STORAGE_ENDPOINT %} --endpoint-url $BACKUP_CUSTOM_STORAGE_ENDPOINT {% endif %} s3 mv $FILENAME s3://$S3_BUCKET_NAME/$BUCKET_PATH/mongo/"]
+          {% if BACKUP_K8S_USE_EPHEMERAL_VOLUMES %}
+          volumes:
+            - name: backup-volume
+              ephemeral:
+                volumeClaimTemplate:
+                  spec:
+                    accessModes:
+                      - ReadWriteOnce
+                    resources:
+                      requests:
+                        storage: {{ BACKUP_K8S_EPHEMERAL_VOLUME_SIZE }}
+          {% endif %}

drydock_backups/patches/k8s-jobs

@@ -0,0 +1 @@
+{{ patch("drydock-multipurpose-jobs") }}