Chop up docker-compose configs

.github/actions/run-docker/action.yml

@@ -1,55 +1,51 @@
 name: 'Docker Run Action'
 description: 'Run a command in a new container'
 inputs:
-  image:
-    description: "The Docker image to run"
+  version:
+    description: 'The version of the image to run'
     required: true
-  options:
-    description: 'Options'
-    required: false
   run:
     description: 'Run command in container'
     required: true
+  services:
+    description: 'List of services to start'
+    required: false
+    default: 'web'
+  compose_file:
+    description: 'The docker-compose file to use'
+    required: false
+    default: 'docker-compose.yml'
 runs:
   using: 'composite'
   steps:
+    - id: id
+      shell: bash
+      run: |
+        echo "id=$(id -u)" >> $GITHUB_OUTPUT
+
     - name: Run Docker Container
       shell: bash
+      env:
+        DOCKER_VERSION: ${{ inputs.version }}
+        COMPOSE_FILE: ${{ inputs.compose_file }}
+        DOCKER_SERVICES: ${{ inputs.services }}
+        HOST_UID: ${{ steps.id.outputs.id }}
       run: |
-        # Export .env file with host user info
-        make -f Makefile-os create_env_file
+        if [[ -z "${{ inputs.run }}" ]]; then
+          echo "run input is required"
+          exit 1
+        fi
 
-        cat <<EOF > exec.sh
-        #!/bin/bash
-        whoami
-        ${{ inputs.run }}
-        EOF
+        # Start the specified services
+        make up
 
-        cat <<EOF > root.sh
-        #!/bin/bash
-        whoami
-        ./docker/entrypoint.sh
-        su -s /bin/bash -c './exec.sh' olympia
+        # Exec the run command in the container
+        # quoted 'EOF' to prevent variable expansion
+        cat <<'EOF' | docker compose exec --user olympia web sh
+          ${{ inputs.run }}
         EOF
 
-        # Make both files executable
-        chmod +x exec.sh
-        chmod +x root.sh
-
-        # Debug info
-        echo "############"
-        cat root.sh
-        echo "############"
-        echo "############"
-        cat exec.sh
-        echo "############"
-
-        # Execute inside docker container
-        cat root.sh | docker run ${{ inputs.options }} \
-          --env-file .env \
-          -i --rm -u 0 \
-          -v $(pwd):/data/olympia \
-          -v ./deps:/deps \
-          -v ./package.json:/deps/package.json \
-          -v ./package-lock.json:/deps/package-lock.json \
-          ${{ inputs.image }} bash
+    - name: Logs
+      shell: bash
+      if: failure()
+      run: docker compose logs

.github/workflows/docs.yml

@@ -25,10 +25,8 @@ jobs:
       - name: Build Docs
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
-          options:
+          version: ${{ steps.build.outputs.version }}
           run: |
-            make update_deps
             make docs
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/extract-locales.yml

@@ -28,8 +28,7 @@ jobs:
       - name: Extract Locales
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
-          options:
+          version: ${{ steps.build.outputs.version }}
           run: |
             make update_deps
             make extract_locales

.github/workflows/verify-docker-image.yml

@@ -100,7 +100,7 @@ jobs:
         id: failure
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
+          version: ${{ steps.build.outputs.version }}
           run: |
             exit 1
         continue-on-error: true
@@ -115,7 +115,7 @@ jobs:
       - name: Check (special characters in command)
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
+          version: ${{ steps.build.outputs.version }}
           run: |
             echo 'this is a question?'
             echo 'a * is born'
@@ -124,8 +124,26 @@ jobs:
       - name: Manage py check
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
-          options:
+          version: ${{ steps.build.outputs.version }}
           run: |
-            make update_deps
             make check
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Run Test
+        uses: ./.github/actions/run-docker
+        with:
+          version: ${{ steps.build.outputs.version }}
+          services: ''
+          run: |
+            pytest \
+              -n auto \
+              -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \
+              -v src/olympia/amo

.gitignore

@@ -56,7 +56,9 @@ deps/*
 private/
 
 # do not ignore the following files
+!docker-compose._*.yml
 !docker-compose.private.yml
+!docker-compose.production.yml
 !private/README.md
 !deps/.keep
 

Makefile-os

@@ -7,6 +7,8 @@ define ENV_VAR
 export $(1) ?= $(or $(shell test -f .env && grep -w $(1) .env | cut -d '=' -f2), $(2))
 endef
 
+# This value can be hardcoded, but should still be synced to the .env file
+override DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 $(eval $(call ENV_VAR,DOCKER_VERSION,$(shell echo local)))
 $(eval $(call ENV_VAR,HOST_UID,$(shell id -u)))
 $(eval $(call ENV_VAR,SUPERUSER_EMAIL,$(shell git config user.email || echo admin@mozilla.com)))
@@ -20,6 +22,7 @@ create_env_file:
 	echo "HOST_UID=${HOST_UID}" >> .env
 	echo "SUPERUSER_EMAIL=${SUPERUSER_EMAIL}" >> .env
 	echo "SUPERUSER_USERNAME=${SUPERUSER_USERNAME}" >> .env
+	echo "DOCKER_MYSQLD_VOLUME=${DOCKER_MYSQLD_VOLUME}" >> .env
 
 ####################################################################################################
 
@@ -30,7 +33,6 @@ DOCKER_OUTPUT ?=
 DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit")
 VERSION_BUILD_URL ?= build
 BUILDX_BAKE_COMMAND := docker buildx bake web
-export DOCKER_MYSQLD_VOLUME = addons-server_data_mysqld
 
 override BACKUPS_DIR = $(shell pwd)/backups
 override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)

docker-compose._base.yml

@@ -0,0 +1,16 @@
+networks:
+  default:
+
+volumes:
+  data_redis:
+  data_elasticsearch:
+  data_mysqld:
+    name: ${DOCKER_MYSQLD_VOLUME:-}
+    external: true
+  data_rabbitmq:
+  storage:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${PWD}/storage

docker-compose._services.yml

@@ -0,0 +1,97 @@
+include:
+  - path:
+      - docker-compose._base.yml
+
+services:
+  nginx:
+    image: nginx
+    volumes:
+      - ./docker/nginx/addons.conf:/etc/nginx/conf.d/addons.conf
+      - ./static:/srv/static
+      - ./site-static:/srv/site-static
+      - storage:/shared_storage/uploads:/srv/user-media
+      - storage:/files:/srv/user-media/addons
+      - storage:/guarded-addons:/srv/user-media/guarded-addons
+      - storage:/sitemaps:/srv/user-media/sitemaps
+    ports:
+      - "80:80"
+    networks:
+      default:
+        aliases:
+          - olympia.test
+    depends_on:
+      - web
+      - addons-frontend
+
+  memcached:
+    image: memcached:1.4
+    # Remove this once we upgrade to a version that provides multi-platform images
+    platform: linux/amd64
+
+  mysqld:
+    image: mysql:8.0
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=yes
+      - MYSQL_DATABASE=olympia
+    ports:
+    - "3306:3306"
+    volumes:
+      - data_mysqld:/var/lib/mysql
+
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.3
+    environment:
+      # Disable all xpack related features to avoid unrelated logging
+      # in docker logs. https://github.com/mozilla/addons-server/issues/8887
+      # This also avoids us to require authentication for local development
+      # which simplifies the setup.
+      - xpack.security.enabled=false
+      - xpack.monitoring.enabled=false
+      - xpack.graph.enabled=false
+      - xpack.watcher.enabled=false
+      - "discovery.type=single-node"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    mem_limit: 2g
+    volumes:
+      - data_elasticsearch:/usr/share/elasticsearch/data
+
+  redis:
+    image: redis:6.2
+    volumes:
+      - data_redis:/data
+
+  rabbitmq:
+    image: rabbitmq:3.12
+    hostname: olympia
+    expose:
+      - "5672"
+    environment:
+      - RABBITMQ_DEFAULT_USER=olympia
+      - RABBITMQ_DEFAULT_PASS=olympia
+      - RABBITMQ_DEFAULT_VHOST=olympia
+    volumes:
+      - data_rabbitmq:/var/lib/rabbitmq
+
+  autograph:
+    image: mozilla/autograph:3.3.2
+    platform: linux/amd64
+    command: /go/bin/autograph -c /data/olympia/scripts/autograph_localdev_config.yaml
+    volumes:
+      - .:/data/olympia
+
+  addons-frontend:
+    <<: *env
+    image: mozilla/addons-frontend:latest
+    platform: linux/amd64
+    environment:
+      # We change the proxy port (which is the main entrypoint) as well as the
+      # webpack port to avoid a conflict in case someone runs both addons-server
+      # and addons-frontend locally, with the frontend configured to access
+      # addons-server locally.
+      - PROXY_PORT=7010
+      - WEBPACK_SERVER_PORT=7011
+    ports:
+      # We need to expose this port so that statics can be fetched (they are
+      # exposed using webpack and not by the node app server).
+      - 7011:7011
+    command: yarn amo:olympia

docker-compose._web.yml

@@ -0,0 +1,45 @@
+include:
+  - path:
+      - docker-compose._base.yml
+
+services:
+  web:
+    environment:
+    - CELERY_BROKER_URL=amqp://olympia:olympia@rabbitmq/olympia
+    - CELERY_RESULT_BACKEND=redis://redis:6379/1
+    - DATABASES_DEFAULT_URL=mysql://root:@mysqld/olympia
+    - ELASTICSEARCH_LOCATION=elasticsearch:9200
+    - MEMCACHE_LOCATION=memcached:11211
+    - MYSQL_DATABASE=olympia
+    - MYSQL_ROOT_PASSWORD=docker
+    - OLYMPIA_SITE_URL=http://olympia.test
+    - PYTHONDONTWRITEBYTECODE=1
+    - PYTHONUNBUFFERED=1
+    - PYTHONBREAKPOINT=ipdb.set_trace
+    - TERM=xterm-256color
+    - CIRCLECI=${CIRCLECI}
+    - HISTFILE=/data/olympia/docker/artifacts/bash_history
+    - HISTSIZE=50000
+    - HISTIGNORE=ls:exit:"cd .."
+    - HISTCONTROL=erasedups
+    # Note: docker compose uses the values exported from .env for HOST_UID if
+    # it exists. ./docker/entrypoint.sh uses this variable to fix
+    # the uid of the olympia user to match the host user id if necessary.
+    - HOST_UID=${HOST_UID:-}
+    # This is the email address of the superuser created when initializing the db
+    - SUPERUSER_EMAIL=${SUPERUSER_EMAIL:-}
+    - SUPERUSER_USERNAME=${SUPERUSER_USERNAME:-}
+    image: mozilla/addons-server:${DOCKER_VERSION:-}
+    # We drop down to a different user through supervisord, but starting as
+    # root allows us to fix the ownership of files generated at image build
+    # time through the ./docker/entrypoint.sh script.
+    user: root
+    platform: linux/amd64
+    entrypoint: ["/data/olympia/docker/entrypoint.sh"]
+    command:
+      - supervisord -n -c /data/olympia/docker/supervisor.conf
+    volumes:
+      - storage:/data/olympia/storage
+    extra_hosts:
+     - "olympia.test:127.0.0.1"
+    restart: on-failure:5

docker-compose.production.yml

@@ -0,0 +1,15 @@
+include:
+  - path:
+      - docker-compose._services.yml
+
+services:
+  web:
+    extends:
+      file: docker-compose._web.yml
+      service: web
+
+  worker:
+    extends:
+      service: web
+    command:
+      - supervisord -n -c /data/olympia/docker/supervisor-celery.conf