From 27d2f8932f9dc9953fdbe9fe7df08d1bf5816582 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Mon, 13 May 2024 12:06:26 +0200 Subject: [PATCH] Make up/down commands to easily recreate local environments --- .gitignore | 2 +- Makefile-os | 70 ++++++++++++++++++++++++++---------- docker-compose._services.yml | 10 ++++-- 3 files changed, 60 insertions(+), 22 deletions(-) diff --git a/.gitignore b/.gitignore index 3f0bc27d242..05c56af3c0a 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,4 @@ docker-cache-new/ version.json # Ignore db backup files -backup-olympia.*.sql +backups diff --git a/Makefile-os b/Makefile-os index 66042e4a1fd..863f6fae233 100644 --- a/Makefile-os +++ b/Makefile-os @@ -7,7 +7,10 @@ DOCKER_PUSH ?= false DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit") VERSION_BUILD_URL ?= build -EXPORT_FILE ?= backup-olympia.$(shell date +%Y%m%d%H%M%S).sql +BACKUPS_DIR = $(shell pwd)/backups +EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S) +RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1) + # Exporting these variables make them default values for docker-compose*.yml files export DOCKER_VERSION ?= local @@ -51,17 +54,30 @@ create_env_file: echo "HOST_UID=${HOST_UID}" >> .env echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env - -.PHONY: db_export -db_export: - @echo "Exporting database to $(EXPORT_FILE)" - docker compose exec mysqld /usr/bin/mysqldump olympia > $(EXPORT_FILE) - -.PHONY: db_restore -db_restore: - @if [ ! -f "$(RESTORE_FILE)" ]; then echo "File $(RESTORE_FILE) does not exist"; exit 1; fi - @echo "Restoring database from $(RESTORE_FILE)" - cat $(RESTORE_FILE) | docker compose exec -T mysqld /usr/bin/mysql olympia + echo "DOCKER_VERSION=${DOCKER_VERSION}" >> .env + echo "DOCKER_TARGET=${DOCKER_TARGET}" >> .env + +.PHONY: data_export +data_export: + # Export docker volumes to $(EXPORT_DIR) + mkdir -p $(EXPORT_DIR) + # Export data_mysqld volume + docker run --rm -v data_mysqld:/data -v $(EXPORT_DIR):/backup alpine tar -cjf /backup/data_mysqld.tar -C /data . + # Export data_elasticsearch volume + docker run --rm -v data_elasticsearch:/data -v $(EXPORT_DIR):/backup alpine tar -cjf /backup/data_elasticsearch.tar -C /data . + # Export data_rabbitmq volume + docker run --rm -v data_rabbitmq:/data -v $(EXPORT_DIR):/backup alpine tar -cjf /backup/data_rabbitmq.tar -C /data . + +.PHONY: data_restore +data_restore: + @[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1) + # Restoring docker volumes from $(RESTORE_DIR) + # Restore data_mysqld volume + docker run --rm -v data_mysqld:/data -v $(RESTORE_DIR):/backup alpine tar -xjf /backup/data_mysqld.tar -C /data + # Restore data_elasticsearch volume + docker run --rm -v data_elasticsearch:/data -v $(RESTORE_DIR):/backup alpine tar -xjf /backup/data_elasticsearch.tar -C /data + # Restore data_rabbitmq volume + docker run --rm -v data_rabbitmq:/data -v $(RESTORE_DIR):/backup alpine tar -xjf /backup/data_rabbitmq.tar -C /data .PHONY: create_docker_builder create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images @@ -70,6 +86,7 @@ create_docker_builder: ## Create a custom builder for buildkit to efficiently bu --driver=docker-container DOCKER_BUILD_ARGS := \ +--file=docker-compose.yml \ --progress=$(DOCKER_PROGRESS) \ --builder=$(DOCKER_BUILDER) \ @@ -92,18 +109,25 @@ docker_compose_config: ## Show the docker compose configuration echo $(DOCKER_BUILD_ARGS) .PHONY: build_docker_image -build_docker_image: create_docker_builder version docker_compose_config ## Build the docker image - docker buildx bake web $(DOCKER_BUILD_ARGS) --print +build_docker_image: create_docker_builder version ## Build the docker image docker buildx bake web $(DOCKER_BUILD_ARGS) +.PHONY: docker_compose_down +docker_compose_down: ## Stop the docker containers + docker compose down --rmi local --remove-orphans + .PHONY: clean_docker -clean_docker: ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this commandUse with caution. - docker compose down --rmi local --volumes --remove-orphans +clean_docker: docker_compose_down ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this command. + docker compose down --volumes docker buildx prune -af rm -rf ./deps/** -.PHONY: initialize_docker -initialize_docker: create_env_file build_docker_image +.PHONY: docker_compose_up +docker_compose_up: ## Start the docker containers + docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS) + +.PHONY: docker_extract_deps +docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount # Run a fresh container from the base image to install deps. Since /deps is # shared via a volume in docker-compose.yml, this installs deps for both web # and worker containers, and does so without requiring the containers to be up. @@ -116,7 +140,15 @@ initialize_docker: create_env_file build_docker_image # mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container # running `update_deps` will install the dependencies in the /deps directory before running docker compose run --rm web make update_deps - docker compose up -d + +.PHONY: up +up: create_env_file version docker_extract_deps docker_compose_up ## Create and start docker compose + +.PHONY: down +down: docker_compose_down ## Stop the docker containers + +.PHONY: initialize_docker +initialize_docker: up docker compose exec --user olympia web make initialize %: ## This directs any other recipe (command) to the web container's make. diff --git a/docker-compose._services.yml b/docker-compose._services.yml index 09e65cc1fd6..4e5c57334a9 100644 --- a/docker-compose._services.yml +++ b/docker-compose._services.yml @@ -38,7 +38,7 @@ services: ports: - "3306:3306" volumes: - - data:/var/lib/mysql + - data_mysqld:/var/lib/mysql elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3 @@ -54,6 +54,8 @@ services: - "discovery.type=single-node" - "ES_JAVA_OPTS=-Xms512m -Xmx512m" mem_limit: 2g + volumes: + - data_elasticsearch:/usr/share/elasticsearch/data redis: image: redis:6.2 @@ -67,6 +69,8 @@ services: - RABBITMQ_DEFAULT_USER=olympia - RABBITMQ_DEFAULT_PASS=olympia - RABBITMQ_DEFAULT_VHOST=olympia + volumes: + - data_rabbitmq:/var/lib/rabbitmq autograph: image: mozilla/autograph:3.3.2 @@ -95,7 +99,9 @@ networks: default: volumes: - data: + data_elasticsearch: + data_mysqld: + data_rabbitmq: storage: driver: local driver_opts: