From 9e16c4a903d1456a1a88c13cc46dbb35f7df6a27 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Wed, 15 May 2024 09:32:42 +0200 Subject: [PATCH 1/3] TMP: refactor buildx bake definition and test --- .github/workflows/verify-docker-image.yml | 78 +++++++++++++++++------ Makefile-os | 25 +++++--- 2 files changed, 73 insertions(+), 30 deletions(-) diff --git a/.github/workflows/verify-docker-image.yml b/.github/workflows/verify-docker-image.yml index a27f7bbdbe1..af53f49e834 100644 --- a/.github/workflows/verify-docker-image.yml +++ b/.github/workflows/verify-docker-image.yml @@ -11,42 +11,80 @@ jobs: strategy: matrix: include: - - expected: { version: local, push: false } - input: { version: '', push: '' } - - expected: { version: version, push: true } - input: { version: version, push: true } + - expected: { version: 'local', push: false, uid: ''} + input: { version: '', push: '', uid: '' } + - expected: { version: version, push: true, uid: '1' } + input: { version: version, push: true, uid: '1' } steps: - uses: actions/checkout@v4 - name: Check Docker Compose (default) id: default - env: - DOCKER_VERSION: ${{ matrix.input.version }} - DOCKER_PUSH: ${{ matrix.input.push }} shell: bash run: | - set -xue - config=$(make docker_compose_config) + set -ue - # Expect image tag is correct - echo $config | grep -q "image: mozilla/addons-server:${{ matrix.expected.version }}" + export SUPERUSER_EMAIL="email" + export SUPERUSER_USERNAME="username" + + if [ -n "${{ matrix.input.version }}" ]; then + export DOCKER_VERSION="${{ matrix.expected.version }}" + fi + + if [ -n "${{ matrix.input.push }}" ]; then + export DOCKER_PUSH="${{ matrix.expected.push }}" + fi + + if [ -n "${{ matrix.input.uid }}" ]; then + export HOST_UID="${{ matrix.expected.uid }}" + fi + + function test_assert() { + echo "Expected: $1" + echo "Actual: $2" + if [ -n "${3:-}" ]; then echo "Message: $3"; fi + if [[ "$1" != "$2" ]]; then exit 1; fi + } + + function test_compose() { + test_assert "$(make docker_compose_config | yq e $1 -)" "$2" "$1" + } + + function test_bake() { + test_assert "$(make docker_build_config | jq -r $1)" "$2" "$1" + } + + test_compose '.services.web.image' "mozilla/addons-server:${{ matrix.expected.version }}" + + expected_id=$(id -u) + if [ -n "${{ matrix.input.uid }}" ]; then expected_id="${{ matrix.expected.uid }}"; fi + test_compose '.services.web.environment.HOST_UID' "$expected_id" + + test_compose '.services.web.environment.SUPERUSER_EMAIL' "$SUPERUSER_EMAIL" + test_compose '.services.web.environment.SUPERUSER_USERNAME' "$SUPERUSER_USERNAME" # Expect docker push args are correct + builder="test_builder" + progress="test_progress" + actual_build_args=$( + make docker_build_args \ + DOCKER_PROGRESS=$progress \ + DOCKER_BUILDER=$builder + ) + expected_build_args="docker buildx bake web --progress=$progress --builder=$builder" + if [[ ${{ matrix.expected.push }} == "true" ]]; then - echo $config | grep -q -- "--push" - echo $config | grep -v -q -- "--load" + expected_build_args="$expected_build_args --push" else - echo $config | grep -v -q -- "--push" - echo $config | grep -q -- "--load" + expected_build_args="$expected_build_args --load" fi - # Expect docker platform is correct - echo $config | grep -q -- "platform: linux/amd64" - echo $config | grep -q -- "\"platforms\": \[ - \"linux/amd64\" - \]" + test_assert "$actual_build_args" "$expected_build_args" + # Expect docker platform is correct + test_compose '.services.web.platform' 'linux/amd64' + test_bake '.target.web.platforms[0]' 'linux/amd64' verify_docker_image: runs-on: ubuntu-latest diff --git a/Makefile-os b/Makefile-os index 93d82e971c3..3f4c4f08524 100644 --- a/Makefile-os +++ b/Makefile-os @@ -7,6 +7,7 @@ DOCKER_PUSH ?= false DOCKER_OUTPUT ?= DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit") VERSION_BUILD_URL ?= build +BUILDX_BAKE_COMMAND := docker buildx bake web export DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld BACKUPS_DIR = $(shell pwd)/backups @@ -79,18 +80,18 @@ create_docker_builder: ## Create a custom builder for buildkit to efficiently bu --name $(DOCKER_BUILDER) \ --driver=docker-container -DOCKER_BUILD_ARGS := \ +BUILDX_BAKE_COMMAND += \ --progress=$(DOCKER_PROGRESS) \ --builder=$(DOCKER_BUILDER) \ ifeq ($(DOCKER_PUSH), true) - DOCKER_BUILD_ARGS += --push + BUILDX_BAKE_COMMAND += --push else - DOCKER_BUILD_ARGS += --load + BUILDX_BAKE_COMMAND += --load endif ifneq ($(DOCKER_OUTPUT),) - DOCKER_BUILD_ARGS += --metadata-file=$(DOCKER_OUTPUT) + BUILDX_BAKE_COMMAND += --metadata-file=$(DOCKER_OUTPUT) endif .PHONY: version @@ -99,15 +100,19 @@ version: ## create version.json file .PHONY: docker_compose_config docker_compose_config: ## Show the docker compose configuration - @echo "version: $(DOCKER_VERSION)" - @echo "push: $(DOCKER_PUSH)" - docker compose config web - docker buildx bake web --print - echo $(DOCKER_BUILD_ARGS) + @docker compose config $(DOCKER_SERVICES) + +.PHONY: docker_build_args +docker_build_args: ## Show the docker build configuration + @echo $(BUILDX_BAKE_COMMAND) + +.PHONY: docker_build_config +docker_build_config: + @$(BUILDX_BAKE_COMMAND) --print .PHONY: build_docker_image build_docker_image: create_docker_builder version ## Build the docker image - docker buildx bake web $(DOCKER_BUILD_ARGS) + $(BUILDX_BAKE_COMMAND) .PHONY: docker_mysqld_volume_create docker_mysqld_volume_create: ## Create the mysqld volume From bc490e537f67359e956aac7f2b198d8d3ac3af11 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Wed, 15 May 2024 09:35:46 +0200 Subject: [PATCH 2/3] TMP: better initialization and organization of make file env vars (docker relevant +) --- .circleci/config.yml | 9 ++++ Makefile-os | 43 +++++++++++-------- docker-compose.yml | 6 +-- tests/test_make.py | 99 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 137 insertions(+), 20 deletions(-) create mode 100644 tests/test_make.py diff --git a/.circleci/config.yml b/.circleci/config.yml index bd8becab5b8..94d83e716a0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -597,6 +597,14 @@ jobs: --ignore src/olympia/versions/ \ --ignore src/olympia/zadmin + make_tests: + <<: *defaults-with-services + steps: + - setup_container + - run: + command: | + pytest -v tests + reviewers-and-zadmin: <<: *defaults-with-autograph steps: @@ -660,6 +668,7 @@ workflows: - codestyle - devhub - main + - make_tests # Uncomment if you want to test the docker build # - build-image - reviewers-and-zadmin diff --git a/Makefile-os b/Makefile-os index 3f4c4f08524..701eb30eb59 100644 --- a/Makefile-os +++ b/Makefile-os @@ -1,7 +1,29 @@ -export HOST_UID := $(shell id -u) +#################################################################################################### +# Our makefile makes use of docker compose commands. Our config files rely on environment variables +# both for passing configuration to the containers as well as configuring the compose file itself. +# Variables referenced in docker-compose*.yml should be read from .env, exported and saved in .env -export DOCKER_BUILDER=container +define ENV_VAR +export $(1) ?= $(or $(shell test -f .env && grep -w $(1) .env | cut -d '=' -f2), $(2)) +endef +$(eval $(call ENV_VAR,DOCKER_VERSION,$(shell echo local))) +$(eval $(call ENV_VAR,HOST_UID,$(shell id -u))) +$(eval $(call ENV_VAR,SUPERUSER_EMAIL,$(shell git config user.email || echo admin@mozilla.com))) +$(eval $(call ENV_VAR,SUPERUSER_USERNAME,$(shell git config user.name || echo admin))) + +# Define the create_env_file target to create the .env file +.PHONY: create_env_file +create_env_file: + @ rm -f .env + echo "DOCKER_VERSION=${DOCKER_VERSION}" >> .env + echo "HOST_UID=${HOST_UID}" >> .env + echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env + echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env + +#################################################################################################### + +DOCKER_BUILDER ?= container DOCKER_PROGRESS ?= auto DOCKER_PUSH ?= false DOCKER_OUTPUT ?= @@ -10,16 +32,10 @@ VERSION_BUILD_URL ?= build BUILDX_BAKE_COMMAND := docker buildx bake web export DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld -BACKUPS_DIR = $(shell pwd)/backups -EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S) +override BACKUPS_DIR = $(shell pwd)/backups +override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S) RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1) -# Exporting these variables make them default values for docker-compose*.yml files -export DOCKER_VERSION ?= local - -# define the username and email for the superuser in the container -SUPERUSER_EMAIL=$(shell git config user.email) -SUPERUSER_USERNAME=$(shell git config user.name) .PHONY: help_redirect help_redirect: @@ -47,13 +63,6 @@ shell: ## connect to a running addons-server docker shell rootshell: ## connect to a running addons-server docker shell with root user docker compose exec --user root web bash -.PHONY: create_env_file -create_env_file: - @ rm -f .env - echo "HOST_UID=${HOST_UID}" >> .env - echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env - echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env - .PHONY: data_export data_export: @ mkdir -p $(EXPORT_DIR) diff --git a/docker-compose.yml b/docker-compose.yml index cc9f75b49e5..974e5ba3261 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -22,13 +22,13 @@ x-env-mapping: &env # the uid of the olympia user to match the host user id if necessary. - HOST_UID=${HOST_UID:-} # This is the email address of the superuser created when initializing the db - - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-admin@mozilla.com} - - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-admin} + - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-} + - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-} services: worker: &worker <<: *env - image: mozilla/addons-server:${DOCKER_VERSION:-local} + image: mozilla/addons-server:${DOCKER_VERSION:-} build: context: . dockerfile: Dockerfile diff --git a/tests/test_make.py b/tests/test_make.py new file mode 100644 index 00000000000..32eb77dffca --- /dev/null +++ b/tests/test_make.py @@ -0,0 +1,99 @@ +import os +import subprocess +from itertools import product + +import pytest + + +options = (False, True) +keys = ('DOCKER_VERSION', 'HOST_UID', 'SUPERUSER_EMAIL', 'SUPERUSER_USERNAME') +root_path = os.path.join(os.path.dirname(__file__), '..') +env_path = os.path.join(root_path, '.env') + + +def clean_env(): + if os.path.exists(env_path): + os.remove(env_path) + + env_vars = os.environ.copy() + for key in keys: + env_vars.pop(key, None) + + return env_vars + + +def read_env(): + if not os.path.exists(env_path): + return {} + with open(env_path) as f: + return dict(line.split('=') for line in f.read().splitlines()) + + +def run_make_command(name, use_file=False, use_env=False, use_args=False): + env = clean_env() + + args = ['make', '-f', 'Makefile-os', 'create_env_file'] + + if use_file: + with open(env_path, 'w') as f: + f.write(f'{name}=file') + if use_env: + env[name] = 'env' + if use_args: + args.append(f'{name}=args') + + # Debug before running the command + print(f'name: {name}') + print(f'use_file: {use_file} use_env: {use_env} use_args: {use_args}') + print(f'command: {args}') + print(f'env: {env.get(name, None)}') + print(f'env_file: {read_env().get(name, None)}') + + command = subprocess.run( + args, env=env, capture_output=True, text=True, cwd=root_path + ) + command.check_returncode() + + result = read_env().get(name, None) + + # Debug after running the command + print(f'result: {result}') + + clean_env() + return result + + +default_values = {key: run_make_command(key) for key in keys} + + +@pytest.mark.parametrize( + 'name,use_file,use_env,use_args', + [ + (name, use_file, use_env, use_args) + for name in keys + for use_file, use_env, use_args in product(options, repeat=3) + ], + ids=[ + f'test_permutations_{name}_use_file:{use_file}_use_env:{use_env}_use_args:{use_args}' + for name in keys + for use_file, use_env, use_args in product(options, repeat=3) + ], +) +def test_permutations(name, use_file, use_env, use_args): + expected_value = default_values.get(name) + + if not expected_value: + raise ValueError(f'expected_value is None for {name}') + + if use_file: + expected_value = 'file' + if use_env: + expected_value = 'env' + if use_args: + expected_value = 'args' + + actual_value = run_make_command( + name, use_file=use_file, use_env=use_env, use_args=use_args + ) + + assert actual_value == expected_value From 288116d7b6abbcfb9bf917dee4f0e923ed317403 Mon Sep 17 00:00:00 2001 From: Kevin Meinhardt Date: Wed, 15 May 2024 13:07:48 +0200 Subject: [PATCH 3/3] Chop up docker-compose configs --- .github/actions/run-docker/action.yml | 72 +++++----- .github/workflows/docs.yml | 4 +- .github/workflows/extract-locales.yml | 3 +- .github/workflows/verify-docker-image.yml | 28 +++- .gitignore | 2 + Makefile-os | 4 +- docker-compose._base.yml | 16 +++ docker-compose._services.yml | 97 +++++++++++++ docker-compose._web.yml | 45 ++++++ docker-compose.production.yml | 15 ++ docker-compose.yml | 164 ++-------------------- 11 files changed, 248 insertions(+), 202 deletions(-) create mode 100644 docker-compose._base.yml create mode 100644 docker-compose._services.yml create mode 100644 docker-compose._web.yml create mode 100644 docker-compose.production.yml diff --git a/.github/actions/run-docker/action.yml b/.github/actions/run-docker/action.yml index bc4821a7c37..57829fd8a12 100644 --- a/.github/actions/run-docker/action.yml +++ b/.github/actions/run-docker/action.yml @@ -1,55 +1,51 @@ name: 'Docker Run Action' description: 'Run a command in a new container' inputs: - image: - description: "The Docker image to run" + version: + description: 'The version of the image to run' required: true - options: - description: 'Options' - required: false run: description: 'Run command in container' required: true + services: + description: 'List of services to start' + required: false + default: 'web' + compose_file: + description: 'The docker-compose file to use' + required: false + default: 'docker-compose.yml' runs: using: 'composite' steps: + - id: id + shell: bash + run: | + echo "id=$(id -u)" >> $GITHUB_OUTPUT + - name: Run Docker Container shell: bash + env: + DOCKER_VERSION: ${{ inputs.version }} + COMPOSE_FILE: ${{ inputs.compose_file }} + DOCKER_SERVICES: ${{ inputs.services }} + HOST_UID: ${{ steps.id.outputs.id }} run: | - # Export .env file with host user info - make -f Makefile-os create_env_file + if [[ -z "${{ inputs.run }}" ]]; then + echo "run input is required" + exit 1 + fi - cat < exec.sh - #!/bin/bash - whoami - ${{ inputs.run }} - EOF + # Start the specified services + make up - cat < root.sh - #!/bin/bash - whoami - ./docker/entrypoint.sh - su -s /bin/bash -c './exec.sh' olympia + # Exec the run command in the container + # quoted 'EOF' to prevent variable expansion + cat <<'EOF' | docker compose exec --user olympia web sh + ${{ inputs.run }} EOF - # Make both files executable - chmod +x exec.sh - chmod +x root.sh - - # Debug info - echo "############" - cat root.sh - echo "############" - echo "############" - cat exec.sh - echo "############" - - # Execute inside docker container - cat root.sh | docker run ${{ inputs.options }} \ - --env-file .env \ - -i --rm -u 0 \ - -v $(pwd):/data/olympia \ - -v ./deps:/deps \ - -v ./package.json:/deps/package.json \ - -v ./package-lock.json:/deps/package-lock.json \ - ${{ inputs.image }} bash + - name: Logs + shell: bash + if: failure() + run: docker compose logs diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 712b76430b6..0a8f902ba18 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -25,10 +25,8 @@ jobs: - name: Build Docs uses: ./.github/actions/run-docker with: - image: ${{ steps.build.outputs.tags }} - options: + version: ${{ steps.build.outputs.version }} run: | - make update_deps make docs - name: Upload artifact uses: actions/upload-pages-artifact@v3 diff --git a/.github/workflows/extract-locales.yml b/.github/workflows/extract-locales.yml index ad5edba4d0c..d72fe20b55a 100644 --- a/.github/workflows/extract-locales.yml +++ b/.github/workflows/extract-locales.yml @@ -28,8 +28,7 @@ jobs: - name: Extract Locales uses: ./.github/actions/run-docker with: - image: ${{ steps.build.outputs.tags }} - options: + version: ${{ steps.build.outputs.version }} run: | make update_deps make extract_locales diff --git a/.github/workflows/verify-docker-image.yml b/.github/workflows/verify-docker-image.yml index af53f49e834..8032cb4dfa0 100644 --- a/.github/workflows/verify-docker-image.yml +++ b/.github/workflows/verify-docker-image.yml @@ -100,7 +100,7 @@ jobs: id: failure uses: ./.github/actions/run-docker with: - image: ${{ steps.build.outputs.tags }} + version: ${{ steps.build.outputs.version }} run: | exit 1 continue-on-error: true @@ -115,7 +115,7 @@ jobs: - name: Check (special characters in command) uses: ./.github/actions/run-docker with: - image: ${{ steps.build.outputs.tags }} + version: ${{ steps.build.outputs.version }} run: | echo 'this is a question?' echo 'a * is born' @@ -124,8 +124,26 @@ jobs: - name: Manage py check uses: ./.github/actions/run-docker with: - image: ${{ steps.build.outputs.tags }} - options: + version: ${{ steps.build.outputs.version }} run: | - make update_deps make check + + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - id: build + uses: ./.github/actions/build-docker + + - name: Run Test + uses: ./.github/actions/run-docker + with: + version: ${{ steps.build.outputs.version }} + services: '' + run: | + pytest \ + -n auto \ + -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \ + -v src/olympia/amo diff --git a/.gitignore b/.gitignore index c3c2b33f80f..eda2262f516 100644 --- a/.gitignore +++ b/.gitignore @@ -56,7 +56,9 @@ deps/* private/ # do not ignore the following files +!docker-compose._*.yml !docker-compose.private.yml +!docker-compose.production.yml !private/README.md !deps/.keep diff --git a/Makefile-os b/Makefile-os index 701eb30eb59..a93fa7031e6 100644 --- a/Makefile-os +++ b/Makefile-os @@ -7,6 +7,8 @@ define ENV_VAR export $(1) ?= $(or $(shell test -f .env && grep -w $(1) .env | cut -d '=' -f2), $(2)) endef +# This value can be hardcoded, but should still be synced to the .env file +override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld $(eval $(call ENV_VAR,DOCKER_VERSION,$(shell echo local))) $(eval $(call ENV_VAR,HOST_UID,$(shell id -u))) $(eval $(call ENV_VAR,SUPERUSER_EMAIL,$(shell git config user.email || echo admin@mozilla.com))) @@ -20,6 +22,7 @@ create_env_file: echo "HOST_UID=${HOST_UID}" >> .env echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env + echo "DOCKER_MYSQLD_VOLUME=${DOCKER_MYSQLD_VOLUME}" >> .env #################################################################################################### @@ -30,7 +33,6 @@ DOCKER_OUTPUT ?= DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit") VERSION_BUILD_URL ?= build BUILDX_BAKE_COMMAND := docker buildx bake web -export DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld override BACKUPS_DIR = $(shell pwd)/backups override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S) diff --git a/docker-compose._base.yml b/docker-compose._base.yml new file mode 100644 index 00000000000..4fcf882b410 --- /dev/null +++ b/docker-compose._base.yml @@ -0,0 +1,16 @@ +networks: + default: + +volumes: + data_redis: + data_elasticsearch: + data_mysqld: + name: ${DOCKER_MYSQLD_VOLUME:-} + external: true + data_rabbitmq: + storage: + driver: local + driver_opts: + type: none + o: bind + device: ${PWD}/storage diff --git a/docker-compose._services.yml b/docker-compose._services.yml new file mode 100644 index 00000000000..a7c5315fb43 --- /dev/null +++ b/docker-compose._services.yml @@ -0,0 +1,97 @@ +include: + - path: + - docker-compose._base.yml + +services: + nginx: + image: nginx + volumes: + - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf + - ./static:/srv/static + - ./site-static:/srv/site-static + - storage:/shared_storage/uploads:/srv/user-media + - storage:/files:/srv/user-media/addons + - storage:/guarded-addons:/srv/user-media/guarded-addons + - storage:/sitemaps:/srv/user-media/sitemaps + ports: + - "80:80" + networks: + default: + aliases: + - olympia.test + depends_on: + - web + - addons-frontend + + memcached: + image: memcached:1.4 + # Remove this once we upgrade to a version that provides multi-platform images + platform: linux/amd64 + + mysqld: + image: mysql:8.0 + environment: + - MYSQL_ALLOW_EMPTY_PASSWORD=yes + - MYSQL_DATABASE=olympia + ports: + - "3306:3306" + volumes: + - data_mysqld:/var/lib/mysql + + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3 + environment: + # Disable all xpack related features to avoid unrelated logging + # in docker logs. https://github.com/mozilla/addons-server/issues/8887 + # This also avoids us to require authentication for local development + # which simplifies the setup. + - xpack.security.enabled=false + - xpack.monitoring.enabled=false + - xpack.graph.enabled=false + - xpack.watcher.enabled=false + - "discovery.type=single-node" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + mem_limit: 2g + volumes: + - data_elasticsearch:/usr/share/elasticsearch/data + + redis: + image: redis:6.2 + volumes: + - data_redis:/data + + rabbitmq: + image: rabbitmq:3.12 + hostname: olympia + expose: + - "5672" + environment: + - RABBITMQ_DEFAULT_USER=olympia + - RABBITMQ_DEFAULT_PASS=olympia + - RABBITMQ_DEFAULT_VHOST=olympia + volumes: + - data_rabbitmq:/var/lib/rabbitmq + + autograph: + image: mozilla/autograph:3.3.2 + platform: linux/amd64 + command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml + volumes: + - .:/data/olympia + + addons-frontend: + <<: *env + image: mozilla/addons-frontend:latest + platform: linux/amd64 + environment: + # We change the proxy port (which is the main entrypoint) as well as the + # webpack port to avoid a conflict in case someone runs both addons-server + # and addons-frontend locally, with the frontend configured to access + # addons-server locally. + - PROXY_PORT=7010 + - WEBPACK_SERVER_PORT=7011 + ports: + # We need to expose this port so that statics can be fetched (they are + # exposed using webpack and not by the node app server). + - 7011:7011 + command: yarn amo:olympia diff --git a/docker-compose._web.yml b/docker-compose._web.yml new file mode 100644 index 00000000000..f74d6b46423 --- /dev/null +++ b/docker-compose._web.yml @@ -0,0 +1,45 @@ +include: + - path: + - docker-compose._base.yml + +services: + web: + environment: + - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia + - CELERY_RESULT_BACKEND=redis://redis:6379/1 + - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia + - ELASTICSEARCH_LOCATION=elasticsearch:9200 + - MEMCACHE_LOCATION=memcached:11211 + - MYSQL_DATABASE=olympia + - MYSQL_ROOT_PASSWORD=docker + - OLYMPIA_SITE_URL=http://olympia.test + - PYTHONDONTWRITEBYTECODE=1 + - PYTHONUNBUFFERED=1 + - PYTHONBREAKPOINT=ipdb.set_trace + - TERM=xterm-256color + - CIRCLECI=${CIRCLECI} + - HISTFILE=/data/olympia/docker/artifacts/bash_history + - HISTSIZE=50000 + - HISTIGNORE=ls:exit:"cd .." + - HISTCONTROL=erasedups + # Note: docker compose uses the values exported from .env for HOST_UID if + # it exists. ./docker/entrypoint.sh uses this variable to fix + # the uid of the olympia user to match the host user id if necessary. + - HOST_UID=${HOST_UID:-} + # This is the email address of the superuser created when initializing the db + - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-} + - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-} + image: mozilla/addons-server:${DOCKER_VERSION:-} + # We drop down to a different user through supervisord, but starting as + # root allows us to fix the ownership of files generated at image build + # time through the ./docker/entrypoint.sh script. + user: root + platform: linux/amd64 + entrypoint: ["/data/olympia/docker/entrypoint.sh"] + command: + - supervisord -n -c /data/olympia/docker/supervisor.conf + volumes: + - storage:/data/olympia/storage + extra_hosts: + - "olympia.test:127.0.0.1" + restart: on-failure:5 diff --git a/docker-compose.production.yml b/docker-compose.production.yml new file mode 100644 index 00000000000..5f27efe4e7f --- /dev/null +++ b/docker-compose.production.yml @@ -0,0 +1,15 @@ +include: + - path: + - docker-compose._services.yml + +services: + web: + extends: + file: docker-compose._web.yml + service: web + + worker: + extends: + service: web + command: + - supervisord -n -c /data/olympia/docker/supervisor-celery.conf diff --git a/docker-compose.yml b/docker-compose.yml index 974e5ba3261..26e34382fb4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,34 +1,12 @@ -x-env-mapping: &env - environment: - - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia - - CELERY_RESULT_BACKEND=redis://redis:6379/1 - - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia - - ELASTICSEARCH_LOCATION=elasticsearch:9200 - - MEMCACHE_LOCATION=memcached:11211 - - MYSQL_DATABASE=olympia - - MYSQL_ROOT_PASSWORD=docker - - OLYMPIA_SITE_URL=http://olympia.test - - PYTHONDONTWRITEBYTECODE=1 - - PYTHONUNBUFFERED=1 - - PYTHONBREAKPOINT=ipdb.set_trace - - TERM=xterm-256color - - CIRCLECI=${CIRCLECI} - - HISTFILE=/data/olympia/docker/artifacts/bash_history - - HISTSIZE=50000 - - HISTIGNORE=ls:exit:"cd .." - - HISTCONTROL=erasedups - # Note: docker compose uses the values exported from .env for HOST_UID if - # it exists. ./docker/entrypoint.sh uses this variable to fix - # the uid of the olympia user to match the host user id if necessary. - - HOST_UID=${HOST_UID:-} - # This is the email address of the superuser created when initializing the db - - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-} - - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-} +include: + - path: + - docker-compose._services.yml services: - worker: &worker - <<: *env - image: mozilla/addons-server:${DOCKER_VERSION:-} + web: + extends: + file: docker-compose._web.yml + service: web build: context: . dockerfile: Dockerfile @@ -39,134 +17,14 @@ services: x-bake: pull: true platforms: linux/amd64 - # We drop down to a different user through supervisord, but starting as - # root allows us to fix the ownership of files generated at image build - # time through the ./docker/entrypoint.sh script. - user: root - platform: linux/amd64 - entrypoint: ["/data/olympia/docker/entrypoint.sh"] - command: - - supervisord -n -c /data/olympia/docker/supervisor-celery.conf volumes: - .:/data/olympia - - storage:/data/olympia/storage - ./deps:/deps - ./package.json:/deps/package.json - ./package-lock.json:/deps/package-lock.json - extra_hosts: - - "olympia.test:127.0.0.1" - restart: on-failure:5 - web: - <<: *worker + worker: + extends: + service: web command: - - supervisord -n -c /data/olympia/docker/supervisor.conf - nginx: - image: nginx - volumes: - - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf - - ./static:/srv/static - - ./site-static:/srv/site-static - - storage:/shared_storage/uploads:/srv/user-media - - storage:/files:/srv/user-media/addons - - storage:/guarded-addons:/srv/user-media/guarded-addons - - storage:/sitemaps:/srv/user-media/sitemaps - ports: - - "80:80" - networks: - default: - aliases: - - olympia.test - depends_on: - - web - - addons-frontend - - memcached: - image: memcached:1.4 - # Remove this once we upgrade to a version that provides multi-platform images - platform: linux/amd64 - - mysqld: - image: mysql:8.0 - environment: - - MYSQL_ALLOW_EMPTY_PASSWORD=yes - - MYSQL_DATABASE=olympia - ports: - - "3306:3306" - volumes: - - data_mysqld:/var/lib/mysql - - elasticsearch: - image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3 - environment: - # Disable all xpack related features to avoid unrelated logging - # in docker logs. https://github.com/mozilla/addons-server/issues/8887 - # This also avoids us to require authentication for local development - # which simplifies the setup. - - xpack.security.enabled=false - - xpack.monitoring.enabled=false - - xpack.graph.enabled=false - - xpack.watcher.enabled=false - - "discovery.type=single-node" - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - mem_limit: 2g - volumes: - - data_elasticsearch:/usr/share/elasticsearch/data - - redis: - image: redis:6.2 - volumes: - - data_redis:/data - - rabbitmq: - image: rabbitmq:3.12 - hostname: olympia - expose: - - "5672" - environment: - - RABBITMQ_DEFAULT_USER=olympia - - RABBITMQ_DEFAULT_PASS=olympia - - RABBITMQ_DEFAULT_VHOST=olympia - volumes: - - data_rabbitmq:/var/lib/rabbitmq - - autograph: - image: mozilla/autograph:3.3.2 - platform: linux/amd64 - command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml - volumes: - - .:/data/olympia - - addons-frontend: - <<: *env - image: mozilla/addons-frontend:latest - platform: linux/amd64 - environment: - # We change the proxy port (which is the main entrypoint) as well as the - # webpack port to avoid a conflict in case someone runs both addons-server - # and addons-frontend locally, with the frontend configured to access - # addons-server locally. - - PROXY_PORT=7010 - - WEBPACK_SERVER_PORT=7011 - ports: - # We need to expose this port so that statics can be fetched (they are - # exposed using webpack and not by the node app server). - - 7011:7011 - command: yarn amo:olympia - -networks: - default: - -volumes: - data_redis: - data_elasticsearch: - data_mysqld: - name: ${DOCKER_MYSQLD_VOLUME:-} - external: true - data_rabbitmq: - storage: - driver: local - driver_opts: - type: none - o: bind - device: ${PWD}/storage + - supervisord -n -c /data/olympia/docker/supervisor-celery.conf