WIP: Add backup and restore commands to the makefile

Makefile

@@ -71,6 +71,8 @@ post_configure: $(post_configure_targets)
 
 ##################### Database
 
+DOCKERIZEWAIT = dockerize -wait tcp://mysql:3306 -timeout 20s
+
 databases: provision-databases migrate provision-oauth2 ## Bootstrap databases
 
 provision-databases: ## Create necessary databases and users
@@ -80,8 +82,8 @@ provision-oauth2: ## Create users for SSO between services
 
 migrate: migrate-openedx migrate-forum $(extra_migrate_targets) ## Perform all database migrations
 migrate-openedx: ## Perform database migrations on LMS/CMS
-	$(DOCKER_COMPOSE_RUN) lms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py lms migrate"
-	$(DOCKER_COMPOSE_RUN) cms bash -c "dockerize -wait tcp://mysql:3306 -timeout 20s && ./manage.py cms migrate"
+	$(DOCKER_COMPOSE_RUN) lms bash -c "$(DOCKERIZEWAIT) && ./manage.py lms migrate"
+	$(DOCKER_COMPOSE_RUN) cms bash -c "$(DOCKERIZEWAIT) && ./manage.py cms migrate"
 	$(MAKE) reindex-courses
 migrate-forum: ## Perform database migrations on discussion forums
 	$(DOCKER_COMPOSE_RUN) forum bash -c "bundle exec rake search:initialize && \
@@ -93,6 +95,55 @@ migrate-xqueue: ## Perform database migrations for the XQueue service
 reindex-courses: ## Refresh course index so they can be found in the LMS search engine
 	$(DOCKER_COMPOSE_RUN) cms ./manage.py cms reindex_course --all --setup
 
+##################### Backup Utilities
+
+ISO_NOW=$(shell date -u "+%Y-%m-%dT%H:%M:%S%z")
+backup-lms: ## Backup the mysql database used for the lms.
+	@echo "Exporting mysql database..."
+	@$(DOCKER_COMPOSE_RUN) lms bash -c "export $(shell cat ${PWD}/config/mysql/auth.env); \
+	$(DOCKERIZEWAIT) && mysqldump -u root -p\$$MYSQL_ROOT_PASSWORD --host mysql openedx > \
+	/openedx/backups/lms$(ISO_NOW).sql 2>/dev/null || true"
+	@echo "Backup Complete!"
+	@echo "Created file: lms$(ISO_NOW).sql"
+
+# We need this conditional to see if mongodb is running.
+# When the mongodb container is not running, we need to run it to backup.
+# When the mongodb container is running we can use the running container.
+backup-cms: ## Backup the MongoDB database used for Studio.
+	@echo "Exporting MongoDB database..."
+	@if [ -z `docker ps -q --no-trunc | grep $$(docker-compose ps -q mongodb)` ]; \
+	then echo "mongodb container not running, starting container..."; \
+	docker-compose run --rm mongodb bash -c " \
+	mongod --smallfiles --nojournal --storageEngine wiredTiger --fork --logpath=/var/log/mongodb/mongod.log && \
+	mongodump --out=/openedx/backups/cms$(ISO_NOW)/dump"; \
+	echo "Terminated container"; \
+	else docker-compose exec mongodb bash -c "mongodump --out=/openedx/backups/cms$(ISO_NOW)/dump"; fi
+	@echo "Backup Complete!"
+	@echo "Created backup: cms$(ISO_NOW)"
+
+restore-lms: backup-lms restore-lms-dangerous## Restor the lms mysql db from a dump on the disk. Specify DUMP. A backup will be made first.
+restore-lms-dangerous:
+	@echo "Restoring from file ${DUMP}"
+	@$(DOCKER_COMPOSE_RUN) lms bash -c "export $(shell cat ${PWD}/config/mysql/auth.env); \
+	if [ -f /openedx/backups/${DUMP} ]; \
+	then echo 'Importing dump file ${DUMP}...'; $(DOCKERIZEWAIT) && mysql --user=root --host=mysql --password-p\$$MYSQL_ROOT_PASSWORD openedx < /openedx/backups/${DUMP} 2>/dev/null || true; \
+	echo 'Restore Complete! Used backup ${DUMP}'; \
+	else echo 'File \`${DUMP}\` does not exist! Nothing imported.'; fi"
+
+# The conditional is the same used in the backup mongodb command. See backup-cms
+restore-cms: backup-cms restore-cms-dangerous ## Restor the cms mongo db from a dump on the disk. Specify DUMP. A backup will be made first.
+restore-cms-dangerous:
+	@echo "Restoring from file ${DUMP}"
+	@if [ -z `docker ps -q --no-trunc | grep $$(docker-compose ps -q mongodb)` ]; \
+	then echo "mongodb container not running, starting container..."; \
+	docker-compose run --rm mongodb bash -c " \
+	mongod --smallfiles --nojournal --storageEngine wiredTiger --fork --logpath=/var/log/mongodb/mongod.log && \
+	mongorestore /openedx/backups/${DUMP}/dump"; \
+	echo "Terminated container"; \
+	else docker-compose exec mongodb bash -c "mongorestore /openedx/backups/${DUMP}/dump"; fi
+	@echo "Restore Complete!"
+	@echo "Used backup: ${DUMP}"
+
 ##################### Static assets
 
 # To collect assets we don't rely on the "paver update_assets" command because

README.md

@@ -197,6 +197,32 @@ Open a python shell in the lms or the cms:
     make lms-python
     make cms-python
 
+### Database backup utilities
+
+Backup the mysql database for the lms:
+
+    make backup-lms
+
+Mysql backups are created with `mysqldump` and backups are stored in the `./backups/lms` directory uncompressed. The dump files are automatically timestamped.
+
+Restore a previous backup to the lms:
+
+    make restore-lms DUMP=<name of dumpfile>
+
+By default a backup is made just before restoration is done. If you do not want to backup first run the `restore-lms-dangerous` command. Any backup made with `mysqldump` should be able to be placed in the `./backups/lms` directory and restored with this command.
+
+Backup the MongoDB database for the cms:
+
+    make backup-cms
+
+MongoDB backups are created with `mongodump` and backups are stored in the `./backups/cms` directory uncompressed. The dump files are automatically timestamped.
+
+Restore a previous backup to the lms:
+
+    make restore-cms DUMP=<name of dumpfile>
+
+By default a backup is made just before restoration is done. If you do not want to backup first run the `restore-cms-dangerous` command. Any backup made with `mongodump` should be able to be placed in the `./backups/cms` directory and restored with this command.
+
 ## For developers
 
 In addition to running Open edX in production, you can use the docker containers for local development. This means you can hack on Open edX without setting up a Virtual Machine. Essentially, this replaces the devstack provided by edX.

docker-compose.yml

@@ -14,6 +14,7 @@ services:
     restart: unless-stopped
     volumes:
       - ./data/mongodb:/data/db
+      - ./data/backups/cms:/openedx/backups
 
   mysql:
     image: mysql:5.6.36
@@ -88,6 +89,7 @@ services:
       - ./config/openedx:/openedx/config
       - ./data/lms:/openedx/data
       - ./data/themes:/openedx/themes
+      - ./data/backups/lms:/openedx/backups
     depends_on:
       - elasticsearch
       - forum