Load service data via portable and persistent data volumes

.circleci/config.yml

@@ -368,6 +368,10 @@ commands:
             echo 'export VERSION_BUILD_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV
             echo 'export DOCKER_PUSH=<< parameters.push >>' >> $BASH_ENV
             echo 'export DOCKER_OUTPUT=<< parameters.output_file >>' >> $BASH_ENV
+      - run:
+          name: Docker build config
+          command: |
+            make docker_compose_config
       - run:
           name: Build docker image and push to repo
           command: |

.gitignore

@@ -66,3 +66,6 @@ docker-cache-new/
 
 # Ignore the version.json as this should be created during the build
 version.json
+
+# Ignore db backup files
+backups

Makefile-docker

@@ -96,9 +96,16 @@ initialize_db: ## create a new database
 	$(PYTHON_COMMAND) manage.py migrate --noinput
 	$(PYTHON_COMMAND) manage.py loaddata initial.json
 	$(PYTHON_COMMAND) manage.py import_prod_versions
-	$(PYTHON_COMMAND) manage.py createsuperuser
+	$(PYTHON_COMMAND) manage.py createsuperuser \
+		--no-input \
+		--username "$(SUPERUSER_USERNAME)" \
+		--email $(SUPERUSER_EMAIL)
 	$(PYTHON_COMMAND) manage.py loaddata zadmin/users
 
+.PHONY: reindex_data
+reindex_data: ## reindex the data in elasticsearch
+	$(PYTHON_COMMAND) manage.py reindex --force --noinput
+
 .PHONY: populate_data
 populate_data: ## populate a new database
 	# reindex --wipe will force the ES mapping to be re-installed. Useful to
@@ -112,9 +119,6 @@ populate_data: ## populate a new database
 	# changes.
 	$(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend
 
-	# Now that addons have been generated, reindex.
-	$(PYTHON_COMMAND) manage.py reindex --force --noinput
-
 .PHONY: update_deps_pip
 update_deps_pip: ## Install pip
 	# Work arounds "Multiple .dist-info directories" issue.
@@ -225,7 +229,7 @@ dbshell: ## connect to a database shell
 	$(PYTHON_COMMAND) ./manage.py dbshell
 
 .PHONY: initialize
-initialize: update_deps initialize_db update_assets populate_data ## init the dependencies, the database, and assets
+initialize: update_deps initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets
 
 .PHONY: reload
 reload: ## force django code reload

Makefile-os

@@ -7,9 +7,19 @@ DOCKER_PUSH ?= false
 DOCKER_OUTPUT ?=
 DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit")
 VERSION_BUILD_URL ?= build
+export DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
+
+BACKUPS_DIR = $(shell pwd)/backups
+EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)
+RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1)
+
 # Exporting these variables make them default values for docker-compose*.yml files
 export DOCKER_VERSION ?= local
 
+# define the username and email for the superuser in the container
+SUPERUSER_EMAIL=$(shell git config user.email)
+SUPERUSER_USERNAME=$(shell git config user.name)
+
 .PHONY: help_redirect
 help_redirect:
 	@$(MAKE) help --no-print-directory
@@ -26,11 +36,7 @@ push_locales: ## extracts and merges translation strings
 	bash ./scripts/push_l10n_extraction.sh $(ARGS)
 
 .PHONY: update_docker
-update_docker: ## update all the docker images
-	docker compose exec --user olympia worker make update_deps
-	docker compose exec --user olympia web make update
-	docker compose restart web
-	docker compose restart worker
+update_docker: data_export up data_restore ## update all the docker images
 
 .PHONY: shell
 shell: ## connect to a running addons-server docker shell
@@ -42,7 +48,26 @@ rootshell: ## connect to a running addons-server docker shell with root user
 
 .PHONY: create_env_file
 create_env_file:
-	echo "HOST_UID=${HOST_UID}" > .env
+	@ rm -f .env
+	echo "HOST_UID=${HOST_UID}" >> .env
+	echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env
+	echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env
+
+.PHONY: data_export
+data_export:
+	@ mkdir -p $(EXPORT_DIR)
+
+	# Extracting mysql database
+	docker compose exec mysqld /usr/bin/mysqldump olympia > $(EXPORT_DIR)/data_mysqld.sql
+
+.PHONY: data_restore
+data_restore:
+	@[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1)
+
+	# Restoring mysql database
+	docker compose exec -T mysqld /usr/bin/mysql olympia < $(RESTORE_DIR)/data_mysqld.sql
+
+	$(MAKE) reindex_data
 
 .PHONY: create_docker_builder
 create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images
@@ -77,18 +102,32 @@ docker_compose_config: ## Show the docker compose configuration
 	echo $(DOCKER_BUILD_ARGS)
 
 .PHONY: build_docker_image
-build_docker_image: create_docker_builder version docker_compose_config ## Build the docker image
-	docker buildx bake web $(DOCKER_BUILD_ARGS) --print
+build_docker_image: create_docker_builder version ## Build the docker image
 	docker buildx bake web $(DOCKER_BUILD_ARGS)
 
+.PHONY: docker_mysqld_volume_create
+docker_mysqld_volume_create: ## Create the mysqld volume
+	docker volume create $(DOCKER_MYSQLD_VOLUME)
+
+.PHONY: docker_mysqld_volume_remove
+docker_mysqld_volume_remove: ## Remove the mysqld volume
+	docker volume rm $(DOCKER_MYSQLD_VOLUME)
+
+.PHONY: docker_compose_down
+docker_compose_down: ## Stop the docker containers
+	docker compose down --rmi local --remove-orphans --volumes
+
 .PHONY: clean_docker
-clean_docker: ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this commandUse with caution.
-	docker compose down --rmi local --volumes --remove-orphans
+clean_docker: docker_compose_down docker_mysqld_volume_remove ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this command.
 	docker buildx prune -af
 	rm -rf ./deps/**
 
-.PHONY: initialize_docker
-initialize_docker: create_env_file build_docker_image
+.PHONY: docker_compose_up
+docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
+	docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS)
+
+.PHONY: docker_extract_deps
+docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount
 # Run a fresh container from the base image to install deps. Since /deps is
 # shared via a volume in docker-compose.yml, this installs deps for both web
 # and worker containers, and does so without requiring the containers to be up.
@@ -101,7 +140,15 @@ initialize_docker: create_env_file build_docker_image
 	# mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container
 	# running `update_deps` will install the dependencies in the /deps directory before running
 	docker compose run --rm web make update_deps
-	docker compose up -d
+
+.PHONY: up
+up: create_env_file version docker_mysqld_volume_create docker_extract_deps docker_compose_up ## Create and start docker compose
+
+.PHONY: down
+down: docker_compose_down ## Stop the docker containers
+
+.PHONY: initialize_docker
+initialize_docker: up
 	docker compose exec --user olympia web make initialize
 
 %: ## This directs any other recipe (command) to the web container's make.

docker-compose.yml

@@ -21,6 +21,9 @@ x-env-mapping: &env
     # it exists. ./docker/entrypoint.sh uses this variable to fix
     # the uid of the olympia user to match the host user id if necessary.
     - HOST_UID=${HOST_UID:-}
+    # This is the email address of the superuser created when initializing the db
+    - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-admin@mozilla.com}
+    - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-admin}
 
 services:
   worker: &worker
@@ -46,6 +49,7 @@ services:
       - supervisord -n -c /data/olympia/docker/supervisor-celery.conf
     volumes:
       - .:/data/olympia
+      - storage:/data/olympia/storage
       - ./deps:/deps
       - ./package.json:/deps/package.json
       - ./package-lock.json:/deps/package-lock.json
@@ -63,10 +67,10 @@ services:
       - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
       - ./static:/srv/static
       - ./site-static:/srv/site-static
-      - ./storage/shared_storage/uploads:/srv/user-media
-      - ./storage/files:/srv/user-media/addons
-      - ./storage/guarded-addons:/srv/user-media/guarded-addons
-      - ./storage/sitemaps:/srv/user-media/sitemaps
+      - storage:/shared_storage/uploads:/srv/user-media
+      - storage:/files:/srv/user-media/addons
+      - storage:/guarded-addons:/srv/user-media/guarded-addons
+      - storage:/sitemaps:/srv/user-media/sitemaps
     ports:
       - "80:80"
     networks:
@@ -89,6 +93,8 @@ services:
       - MYSQL_DATABASE=olympia
     ports:
     - "3306:3306"
+    volumes:
+      - data_mysqld:/var/lib/mysql
 
   elasticsearch:
     image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3
@@ -104,9 +110,13 @@ services:
       - "discovery.type=single-node"
       - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
     mem_limit: 2g
+    volumes:
+      - data_elasticsearch:/usr/share/elasticsearch/data
 
   redis:
     image: redis:6.2
+    volumes:
+      - data_redis:/data
 
   rabbitmq:
     image: rabbitmq:3.12
@@ -117,6 +127,8 @@ services:
       - RABBITMQ_DEFAULT_USER=olympia
       - RABBITMQ_DEFAULT_PASS=olympia
       - RABBITMQ_DEFAULT_VHOST=olympia
+    volumes:
+      - data_rabbitmq:/var/lib/rabbitmq
 
   autograph:
     image: mozilla/autograph:3.3.2
@@ -144,3 +156,17 @@ services:
 
 networks:
   default:
+
+volumes:
+  data_redis:
+  data_elasticsearch:
+  data_mysqld:
+    name: ${DOCKER_MYSQLD_VOLUME:-}
+    external: true
+  data_rabbitmq:
+  storage:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}/storage