From 29b85e7f5b7db12d790d391b4f3a0d676ba1a1ac Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Mon, 13 May 2024 09:40:22 +0200 Subject: [PATCH 1/7] Infer superuser credentials from git config --- Makefile-docker | 5 ++++- Makefile-os | 9 ++++++++- docker-compose.yml | 3 +++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/Makefile-docker b/Makefile-docker index 3bb64f42f5e..60b98666713 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -96,7 +96,10 @@ initialize_db: ## create a new database $(PYTHON_COMMAND) manage.py migrate --noinput $(PYTHON_COMMAND) manage.py loaddata initial.json $(PYTHON_COMMAND) manage.py import_prod_versions - $(PYTHON_COMMAND) manage.py createsuperuser + $(PYTHON_COMMAND) manage.py createsuperuser \ + --no-input \ + --username "$(SUPERUSER_USERNAME)" \ + --email $(SUPERUSER_EMAIL) $(PYTHON_COMMAND) manage.py loaddata zadmin/users .PHONY: populate_data diff --git a/Makefile-os b/Makefile-os index bbf202f5505..e85d4686ed0 100644 --- a/Makefile-os +++ b/Makefile-os @@ -10,6 +10,10 @@ VERSION_BUILD_URL ?= build # Exporting these variables make them default values for docker-compose*.yml files export DOCKER_VERSION ?= local +# define the username and email for the superuser in the container +SUPERUSER_EMAIL=$(shell git config user.email) +SUPERUSER_USERNAME=$(shell git config user.name) + .PHONY: help_redirect help_redirect: @$(MAKE) help --no-print-directory @@ -42,7 +46,10 @@ rootshell: ## connect to a running addons-server docker shell with root user .PHONY: create_env_file create_env_file: - echo "HOST_UID=${HOST_UID}" > .env + @ rm -f .env + echo "HOST_UID=${HOST_UID}" >> .env + echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env + echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env .PHONY: create_docker_builder create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images diff --git a/docker-compose.yml b/docker-compose.yml index 96f1db251ee..d5fb42e5428 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,9 @@ x-env-mapping: &env # it exists. ./docker/entrypoint.sh uses this variable to fix # the uid of the olympia user to match the host user id if necessary. - HOST_UID=${HOST_UID:-} + # This is the email address of the superuser created when initializing the db + - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-admin@mozilla.com} + - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-admin} services: worker: &worker From a662a8241d71902dad472386aa726a80637c8cac Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Mon, 13 May 2024 17:41:04 +0200 Subject: [PATCH 2/7] Load service data via portable and persistent data volumes Load storage via persistent data volume (host bound) Make up/down commands to easily recreate local environments --- .circleci/config.yml | 4 +++ .gitignore | 4 +++ Makefile-docker | 9 ++++--- Makefile-os | 60 +++++++++++++++++++++++++++++++++++--------- docker-compose.yml | 26 ++++++++++++++++--- 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 84ce49a1c6b..bd8becab5b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -368,6 +368,10 @@ commands: echo 'export VERSION_BUILD_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV echo 'export DOCKER_PUSH=<< parameters.push >>' >> $BASH_ENV echo 'export DOCKER_OUTPUT=<< parameters.output_file >>' >> $BASH_ENV + - run: + name: Docker build config + command: | + make docker_compose_config - run: name: Build docker image and push to repo command: | diff --git a/.gitignore b/.gitignore index 02f32301335..a5acbba471f 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ private/ # do not ignore the following files !docker-compose.private.yml +!docker-compose.data.yml !private/README.md !deps/.keep @@ -66,3 +67,6 @@ docker-cache-new/ # Ignore the version.json as this should be created during the build version.json + +# Ignore db backup files +backups diff --git a/Makefile-docker b/Makefile-docker index 60b98666713..8940a0a589a 100644 --- a/Makefile-docker +++ b/Makefile-docker @@ -102,6 +102,10 @@ initialize_db: ## create a new database --email $(SUPERUSER_EMAIL) $(PYTHON_COMMAND) manage.py loaddata zadmin/users +.PHONY: reindex_data +reindex_data: ## reindex the data in elasticsearch + $(PYTHON_COMMAND) manage.py reindex --force --noinput + .PHONY: populate_data populate_data: ## populate a new database # reindex --wipe will force the ES mapping to be re-installed. Useful to @@ -115,9 +119,6 @@ populate_data: ## populate a new database # changes. $(PYTHON_COMMAND) manage.py generate_default_addons_for_frontend - # Now that addons have been generated, reindex. - $(PYTHON_COMMAND) manage.py reindex --force --noinput - .PHONY: update_deps_pip update_deps_pip: ## Install pip # Work arounds "Multiple .dist-info directories" issue. @@ -228,7 +229,7 @@ dbshell: ## connect to a database shell $(PYTHON_COMMAND) ./manage.py dbshell .PHONY: initialize -initialize: update_deps initialize_db update_assets populate_data ## init the dependencies, the database, and assets +initialize: update_deps initialize_db update_assets populate_data reindex_data ## init the dependencies, the database, and assets .PHONY: reload reload: ## force django code reload diff --git a/Makefile-os b/Makefile-os index e85d4686ed0..672108ab3e7 100644 --- a/Makefile-os +++ b/Makefile-os @@ -7,6 +7,11 @@ DOCKER_PUSH ?= false DOCKER_OUTPUT ?= DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit") VERSION_BUILD_URL ?= build + +BACKUPS_DIR = $(shell pwd)/backups +EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S) +RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1) + # Exporting these variables make them default values for docker-compose*.yml files export DOCKER_VERSION ?= local @@ -30,11 +35,7 @@ push_locales: ## extracts and merges translation strings bash ./scripts/push_l10n_extraction.sh $(ARGS) .PHONY: update_docker -update_docker: ## update all the docker images - docker compose exec --user olympia worker make update_deps - docker compose exec --user olympia web make update - docker compose restart web - docker compose restart worker +update_docker: data_export up data_restore ## update all the docker images .PHONY: shell shell: ## connect to a running addons-server docker shell @@ -51,6 +52,26 @@ create_env_file: echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env +SERVICES_PATHS := \ + "elasticsearch:/usr/share/elasticsearch/data" \ + "rabbitmq:/var/lib/rabbitmq" + +.PHONY: data_export +data_export: + @ mkdir -p $(EXPORT_DIR) + + # Extracting mysql database + docker compose exec mysqld /usr/bin/mysqldump olympia > $(EXPORT_DIR)/data_mysqld.sql + +.PHONY: data_restore +data_restore: + @[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1) + + # Restoring mysql database + docker compose exec -T mysqld /usr/bin/mysql olympia < $(RESTORE_DIR)/data_mysqld.sql + + $(MAKE) reindex_data + .PHONY: create_docker_builder create_docker_builder: ## Create a custom builder for buildkit to efficiently build local images docker buildx use $(DOCKER_BUILDER) 2>/dev/null || docker buildx create \ @@ -84,18 +105,25 @@ docker_compose_config: ## Show the docker compose configuration echo $(DOCKER_BUILD_ARGS) .PHONY: build_docker_image -build_docker_image: create_docker_builder version docker_compose_config ## Build the docker image - docker buildx bake web $(DOCKER_BUILD_ARGS) --print +build_docker_image: create_docker_builder version ## Build the docker image docker buildx bake web $(DOCKER_BUILD_ARGS) +.PHONY: docker_compose_down +docker_compose_down: ## Stop the docker containers + docker compose down --rmi local --remove-orphans + .PHONY: clean_docker -clean_docker: ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this commandUse with caution. - docker compose down --rmi local --volumes --remove-orphans +clean_docker: docker_compose_down ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this command. + docker compose down --volumes docker buildx prune -af rm -rf ./deps/** -.PHONY: initialize_docker -initialize_docker: create_env_file build_docker_image +.PHONY: docker_compose_up +docker_compose_up: ## Start the docker containers + docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS) + +.PHONY: docker_extract_deps +docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount # Run a fresh container from the base image to install deps. Since /deps is # shared via a volume in docker-compose.yml, this installs deps for both web # and worker containers, and does so without requiring the containers to be up. @@ -108,7 +136,15 @@ initialize_docker: create_env_file build_docker_image # mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container # running `update_deps` will install the dependencies in the /deps directory before running docker compose run --rm web make update_deps - docker compose up -d + +.PHONY: up +up: create_env_file version docker_extract_deps docker_compose_up ## Create and start docker compose + +.PHONY: down +down: docker_compose_down ## Stop the docker containers + +.PHONY: initialize_docker +initialize_docker: up docker compose exec --user olympia web make initialize %: ## This directs any other recipe (command) to the web container's make. diff --git a/docker-compose.yml b/docker-compose.yml index d5fb42e5428..600f20141a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,6 +49,7 @@ services: - supervisord -n -c /data/olympia/docker/supervisor-celery.conf volumes: - .:/data/olympia + - storage:/data/olympia/storage - ./deps:/deps - ./package.json:/deps/package.json - ./package-lock.json:/deps/package-lock.json @@ -66,10 +67,10 @@ services: - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf - ./static:/srv/static - ./site-static:/srv/site-static - - ./storage/shared_storage/uploads:/srv/user-media - - ./storage/files:/srv/user-media/addons - - ./storage/guarded-addons:/srv/user-media/guarded-addons - - ./storage/sitemaps:/srv/user-media/sitemaps + - storage:/shared_storage/uploads:/srv/user-media + - storage:/files:/srv/user-media/addons + - storage:/guarded-addons:/srv/user-media/guarded-addons + - storage:/sitemaps:/srv/user-media/sitemaps ports: - "80:80" networks: @@ -92,6 +93,8 @@ services: - MYSQL_DATABASE=olympia ports: - "3306:3306" + volumes: + - data_mysqld:/var/lib/mysql elasticsearch: image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3 @@ -107,6 +110,8 @@ services: - "discovery.type=single-node" - "ES_JAVA_OPTS=-Xms512m -Xmx512m" mem_limit: 2g + volumes: + - data_elasticsearch:/usr/share/elasticsearch/data redis: image: redis:6.2 @@ -120,6 +125,8 @@ services: - RABBITMQ_DEFAULT_USER=olympia - RABBITMQ_DEFAULT_PASS=olympia - RABBITMQ_DEFAULT_VHOST=olympia + volumes: + - data_rabbitmq:/var/lib/rabbitmq autograph: image: mozilla/autograph:3.3.2 @@ -147,3 +154,14 @@ services: networks: default: + +volumes: + data_elasticsearch: + data_mysqld: + data_rabbitmq: + storage: + driver: local + driver_opts: + type: none + o: bind + device: ${PWD}/storage From fab46e3240cd4a1f59eb7bfacc65eb48968753eb Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Tue, 14 May 2024 10:47:28 +0200 Subject: [PATCH 3/7] TMP: add redis volume too --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 600f20141a7..5a007e94da2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,6 +115,8 @@ services: redis: image: redis:6.2 + volumes: + - data_redis:/data rabbitmq: image: rabbitmq:3.12 @@ -156,6 +158,7 @@ networks: default: volumes: + data_redis: data_elasticsearch: data_mysqld: data_rabbitmq: From 17ed3aaa669bacf032a29a4c556c385c65092f6d Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Tue, 14 May 2024 15:10:45 +0200 Subject: [PATCH 4/7] TMP: better control over volume state --- Makefile-os | 18 +++++++++++++----- docker-compose.yml | 2 ++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Makefile-os b/Makefile-os index 672108ab3e7..38a80b2114b 100644 --- a/Makefile-os +++ b/Makefile-os @@ -7,6 +7,7 @@ DOCKER_PUSH ?= false DOCKER_OUTPUT ?= DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit") VERSION_BUILD_URL ?= build +export DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld BACKUPS_DIR = $(shell pwd)/backups EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S) @@ -108,18 +109,25 @@ docker_compose_config: ## Show the docker compose configuration build_docker_image: create_docker_builder version ## Build the docker image docker buildx bake web $(DOCKER_BUILD_ARGS) +.PHONY: docker_mysqld_volume_create +docker_mysqld_volume_create: ## Create the mysqld volume + docker volume create $(DOCKER_MYSQLD_VOLUME) + +.PHONY: docker_mysqld_volume_remove +docker_mysqld_volume_remove: ## Remove the mysqld volume + docker volume rm $(DOCKER_MYSQLD_VOLUME) + .PHONY: docker_compose_down docker_compose_down: ## Stop the docker containers - docker compose down --rmi local --remove-orphans + docker compose down --rmi local --remove-orphans --volumes .PHONY: clean_docker -clean_docker: docker_compose_down ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this command. - docker compose down --volumes +clean_docker: docker_compose_down docker_mysqld_volume_remove ## Clean up docker containers, images, caches, volumes and local cache directories. Use with caution. To restart the app run make initialize_docker after this command. docker buildx prune -af rm -rf ./deps/** .PHONY: docker_compose_up -docker_compose_up: ## Start the docker containers +docker_compose_up: docker_mysqld_volume_create ## Start the docker containers docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS) .PHONY: docker_extract_deps @@ -138,7 +146,7 @@ docker_extract_deps: ## Extract dependencies from the docker image to a local vo docker compose run --rm web make update_deps .PHONY: up -up: create_env_file version docker_extract_deps docker_compose_up ## Create and start docker compose +up: create_env_file version docker_mysqld_volume_create docker_extract_deps docker_compose_up ## Create and start docker compose .PHONY: down down: docker_compose_down ## Stop the docker containers diff --git a/docker-compose.yml b/docker-compose.yml index 5a007e94da2..cc9f75b49e5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -161,6 +161,8 @@ volumes: data_redis: data_elasticsearch: data_mysqld: + name: ${DOCKER_MYSQLD_VOLUME:-} + external: true data_rabbitmq: storage: driver: local From 1acf8b9a73b5f798ebfa51e5dd52bb613e2b59f1 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Tue, 14 May 2024 15:19:23 +0200 Subject: [PATCH 5/7] TMP: remove dead code --- Makefile-os | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Makefile-os b/Makefile-os index 38a80b2114b..d2e0420ff51 100644 --- a/Makefile-os +++ b/Makefile-os @@ -53,10 +53,6 @@ create_env_file: echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env -SERVICES_PATHS := \ - "elasticsearch:/usr/share/elasticsearch/data" \ - "rabbitmq:/var/lib/rabbitmq" - .PHONY: data_export data_export: @ mkdir -p $(EXPORT_DIR) From 49946ff5d85e2c3c3c90b85900cb74a18c40ddfb Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Tue, 14 May 2024 15:20:53 +0200 Subject: [PATCH 6/7] TMP: more dead code gone --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index a5acbba471f..c3c2b33f80f 100644 --- a/.gitignore +++ b/.gitignore @@ -57,7 +57,6 @@ private/ # do not ignore the following files !docker-compose.private.yml -!docker-compose.data.yml !private/README.md !deps/.keep From 29753a5ac6c3ff65128cf7cc1070b7ecf532f80e Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Tue, 14 May 2024 15:55:18 +0200 Subject: [PATCH 7/7] TMP: add wait for mysql before restoring snapshots --- Makefile-os | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile-os b/Makefile-os index d2e0420ff51..93d82e971c3 100644 --- a/Makefile-os +++ b/Makefile-os @@ -64,6 +64,10 @@ data_export: data_restore: @[ -d $(RESTORE_DIR) ] || (echo "Directory $(RESTORE_DIR) does not exist" && exit 1) + # Wait for MySQL server to be ready + docker compose exec mysqld bash \ + -c 'while ! mysqladmin ping --silent; do echo "waiting"; sleep 1; done' + # Restoring mysql database docker compose exec -T mysqld /usr/bin/mysql olympia < $(RESTORE_DIR)/data_mysqld.sql