Make up/down commands to easily recreate local environments

.gitignore

@@ -70,4 +70,4 @@ docker-cache-new/
 version.json
 
 # Ignore db backup files
-backup-olympia.*.sql
+backups

Makefile-os

@@ -7,7 +7,10 @@ DOCKER_PUSH ?= false
 DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit")
 VERSION_BUILD_URL ?= build
 
-EXPORT_FILE ?= backup-olympia.$(shell date +%Y%m%d%H%M%S).sql
+BACKUPS_DIR = $(shell pwd)/backups
+EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)
+RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1)
+
 # Exporting these variables make them default values for docker-compose*.yml files
 export DOCKER_VERSION ?= local
 
@@ -51,17 +54,30 @@ create_env_file:
 	echo "HOST_UID=${HOST_UID}" >> .env
 	echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env
 	echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env
-
-.PHONY: db_export
-db_export:
-	@echo "Exporting database to $(EXPORT_FILE)"
-	docker compose exec mysqld /usr/bin/mysqldump olympia > $(EXPORT_FILE)
-
-.PHONY: db_restore
-db_restore:
-	@if [ ! -f "$(RESTORE_FILE)" ]; then echo "File $(RESTORE_FILE) does not exist"; exit 1; fi
-	@echo "Restoring database from $(RESTORE_FILE)"
-	cat $(RESTORE_FILE) | docker compose exec -T mysqld /usr/bin/mysql olympia
+	echo "DOCKER_VERSION=${DOCKER_VERSION}" >> .env
+	echo "DOCKER_TARGET=${DOCKER_TARGET}" >> .env
+
+.PHONY: data_export
+data_export:
+	# Export docker volumes to $(EXPORT_DIR)
+	mkdir -p $(EXPORT_DIR)
+	# Export data_mysqld volume
+	docker run --rm -v data_mysqld:/data -v $(EXPORT_DIR):/backup alpine tar -cjf /backup/data_mysqld.tar -C /data .
+	# Export data_elasticsearch volume
+	docker run --rm -v data_elasticsearch:/data -v $(EXPORT_DIR):/backup alpine tar -cjf /backup/data_elasticsearch.tar -C /data .
+	# Export data_rabbitmq volume
+	docker run --rm -v data_rabbitmq:/data -v $(EXPORT_DIR):/backup alpine tar -cjf /backup/data_rabbitmq.tar -C /data .
+
+.PHONY: data_restore
+data_restore:
+	@[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1)
+	# Restoring docker volumes from $(RESTORE_DIR)
+	# Restore data_mysqld volume
+	docker run --rm -v data_mysqld:/data -v $(RESTORE_DIR):/backup alpine tar -xjf /backup/data_mysqld.tar -C /data
+	# Restore data_elasticsearch volume
+	docker run --rm -v data_elasticsearch:/data -v $(RESTORE_DIR):/backup alpine tar -xjf /backup/data_elasticsearch.tar -C /data
+	# Restore data_rabbitmq volume
+	docker run --rm -v data_rabbitmq:/data -v $(RESTORE_DIR):/backup alpine tar -xjf /backup/data_rabbitmq.tar -C /data
 
 .PHONY: create_docker_builder
 create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images
@@ -70,6 +86,7 @@ create_docker_builder: ## Create a custom builder for buildkit to efficiently bu
 		--driver=docker-container
 
 DOCKER_BUILD_ARGS := \
+--file=docker-compose.yml \
 --progress=$(DOCKER_PROGRESS) \
 --builder=$(DOCKER_BUILDER) \
 
@@ -92,18 +109,25 @@ docker_compose_config: ## Show the docker compose configuration
 	echo $(DOCKER_BUILD_ARGS)
 
 .PHONY: build_docker_image
-build_docker_image: create_docker_builder version docker_compose_config ## Build the docker image
-	docker buildx bake web $(DOCKER_BUILD_ARGS) --print
+build_docker_image: create_docker_builder version ## Build the docker image
 	docker buildx bake web $(DOCKER_BUILD_ARGS)
 
+.PHONY: docker_compose_down
+docker_compose_down: ## Stop the docker containers
+	docker compose down --rmi local --remove-orphans
+
 .PHONY: clean_docker
-clean_docker: ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this commandUse with caution.
-	docker compose down --rmi local --volumes --remove-orphans
+clean_docker: docker_compose_down ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this command.
+	docker compose down --volumes
 	docker buildx prune -af
 	rm -rf ./deps/**
 
-.PHONY: initialize_docker
-initialize_docker: create_env_file build_docker_image
+.PHONY: docker_compose_up
+docker_compose_up: ## Start the docker containers
+	docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS)
+
+.PHONY: docker_extract_deps
+docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount
 # Run a fresh container from the base image to install deps. Since /deps is
 # shared via a volume in docker-compose.yml, this installs deps for both web
 # and worker containers, and does so without requiring the containers to be up.
@@ -116,7 +140,15 @@ initialize_docker: create_env_file build_docker_image
 	# mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container
 	# running `update_deps` will install the dependencies in the /deps directory before running
 	docker compose run --rm web make update_deps
-	docker compose up -d
+
+.PHONY: up
+up: create_env_file version docker_extract_deps docker_compose_up ## Create and start docker compose
+
+.PHONY: down
+down: docker_compose_down ## Stop the docker containers
+
+.PHONY: initialize_docker
+initialize_docker: up
 	docker compose exec --user olympia web make initialize
 
 %: ## This directs any other recipe (command) to the web container's make.

docker-compose._services.yml

@@ -38,7 +38,7 @@ services:
     ports:
     - "3306:3306"
     volumes:
-      - data:/var/lib/mysql
+      - data_mysqld:/var/lib/mysql
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3
@@ -54,6 +54,8 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 2g
+    volumes:
+      - data_elasticsearch:/usr/share/elasticsearch/data
 
   redis:
     image: redis:6.2
@@ -67,6 +69,8 @@ services:
       - RABBITMQ_DEFAULT_USER=olympia
       - RABBITMQ_DEFAULT_PASS=olympia
       - RABBITMQ_DEFAULT_VHOST=olympia
+    volumes:
+      - data_rabbitmq:/var/lib/rabbitmq
 
   autograph:
     image: mozilla/autograph:3.3.2
@@ -95,7 +99,9 @@ networks:
   default:
 
 volumes:
-  data:
+  data_elasticsearch:
+  data_mysqld:
+  data_rabbitmq:
   storage:
     driver: local
     driver_opts: