Use docker compose for building and running container (local and CI) (github action and circleCI)

.circleci/config.yml

@@ -597,6 +597,14 @@ jobs:
               --ignore src/olympia/versions/ \
               --ignore src/olympia/zadmin
 
+  make_tests:
+    <<: *defaults-with-services
+    steps:
+      - setup_container
+      - run:
+          command: |
+            pytest -v tests
+
   reviewers-and-zadmin:
     <<: *defaults-with-autograph
     steps:
@@ -660,6 +668,7 @@ workflows:
       - codestyle
       - devhub
       - main
+      - make_tests
       # Uncomment if you want to test the docker build
       # - build-image
       - reviewers-and-zadmin

.github/actions/run-docker/action.yml

@@ -1,55 +1,51 @@
 name: 'Docker Run Action'
 description: 'Run a command in a new container'
 inputs:
-  image:
-    description: "The Docker image to run"
+  version:
+    description: 'The version of the image to run'
     required: true
-  options:
-    description: 'Options'
-    required: false
   run:
     description: 'Run command in container'
     required: true
+  services:
+    description: 'List of services to start'
+    required: false
+    default: 'web'
+  compose_file:
+    description: 'The docker-compose file to use'
+    required: false
+    default: 'docker-compose.yml'
 runs:
   using: 'composite'
   steps:
+    - id: id
+      shell: bash
+      run: |
+        echo "id=$(id -u)" >> $GITHUB_OUTPUT
+
     - name: Run Docker Container
       shell: bash
+      env:
+        DOCKER_VERSION: ${{ inputs.version }}
+        COMPOSE_FILE: ${{ inputs.compose_file }}
+        DOCKER_SERVICES: ${{ inputs.services }}
+        HOST_UID: ${{ steps.id.outputs.id }}
       run: |
-        # Export .env file with host user info
-        make -f Makefile-os create_env_file
+        if [[ -z "${{ inputs.run }}" ]]; then
+          echo "run input is required"
+          exit 1
+        fi
 
-        cat <<EOF > exec.sh
-        #!/bin/bash
-        whoami
-        ${{ inputs.run }}
-        EOF
+        # Start the specified services
+        make up
 
-        cat <<EOF > root.sh
-        #!/bin/bash
-        whoami
-        ./docker/entrypoint.sh
-        su -s /bin/bash -c './exec.sh' olympia
+        # Exec the run command in the container
+        # quoted 'EOF' to prevent variable expansion
+        cat <<'EOF' | docker compose exec --user olympia web sh
+          ${{ inputs.run }}
         EOF
 
-        # Make both files executable
-        chmod +x exec.sh
-        chmod +x root.sh
-
-        # Debug info
-        echo "############"
-        cat root.sh
-        echo "############"
-        echo "############"
-        cat exec.sh
-        echo "############"
-
-        # Execute inside docker container
-        cat root.sh | docker run ${{ inputs.options }} \
-          --env-file .env \
-          -i --rm -u 0 \
-          -v $(pwd):/data/olympia \
-          -v ./deps:/deps \
-          -v ./package.json:/deps/package.json \
-          -v ./package-lock.json:/deps/package-lock.json \
-          ${{ inputs.image }} bash
+    - name: Logs
+      shell: bash
+      if: failure()
+      run: docker compose logs

.github/workflows/docs.yml

@@ -25,10 +25,8 @@ jobs:
       - name: Build Docs
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
-          options:
+          version: ${{ steps.build.outputs.version }}
           run: |
-            make update_deps
             make docs
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/extract-locales.yml

@@ -28,8 +28,7 @@ jobs:
       - name: Extract Locales
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
-          options:
+          version: ${{ steps.build.outputs.version }}
           run: |
             make update_deps
             make extract_locales

.github/workflows/verify-docker-image.yml

@@ -11,42 +11,80 @@ jobs:
     strategy:
       matrix:
         include:
-          - expected: { version: local, push: false }
-            input: { version: '', push: '' }
-          - expected: { version: version, push: true }
-            input: { version: version, push: true }
+          - expected: { version: 'local', push: false, uid: ''}
+            input: { version: '', push: '', uid: '' }
+          - expected: { version: version, push: true, uid: '1' }
+            input: { version: version, push: true, uid: '1' }
 
     steps:
       - uses: actions/checkout@v4
 
       - name: Check Docker Compose (default)
         id: default
-        env:
-          DOCKER_VERSION: ${{ matrix.input.version }}
-          DOCKER_PUSH: ${{ matrix.input.push }}
         shell: bash
         run: |
-          set -xue
-          config=$(make docker_compose_config)
+          set -ue
 
-          # Expect image tag is correct
-          echo $config | grep -q "image: mozilla/addons-server:${{ matrix.expected.version }}"
+          export SUPERUSER_EMAIL="email"
+          export SUPERUSER_USERNAME="username"
+
+          if [ -n "${{ matrix.input.version }}" ]; then
+            export DOCKER_VERSION="${{ matrix.expected.version }}"
+          fi
+
+          if [ -n "${{ matrix.input.push }}" ]; then
+            export DOCKER_PUSH="${{ matrix.expected.push }}"
+          fi
+
+          if [ -n "${{ matrix.input.uid }}" ]; then
+            export HOST_UID="${{ matrix.expected.uid }}"
+          fi
+
+          function test_assert() {
+            echo "Expected: $1"
+            echo "Actual: $2"
+            if [ -n "${3:-}" ]; then echo "Message: $3"; fi
+            if [[ "$1" != "$2" ]]; then exit 1; fi
+          }
+
+          function test_compose() {
+            test_assert "$(make docker_compose_config | yq e $1 -)" "$2" "$1"
+          }
+
+          function test_bake() {
+            test_assert "$(make docker_build_config | jq -r $1)" "$2" "$1"
+          }
+
+          test_compose '.services.web.image' "mozilla/addons-server:${{ matrix.expected.version }}"
+
+          expected_id=$(id -u)
+          if [ -n "${{ matrix.input.uid }}" ]; then expected_id="${{ matrix.expected.uid }}"; fi
+          test_compose '.services.web.environment.HOST_UID' "$expected_id"
+
+          test_compose '.services.web.environment.SUPERUSER_EMAIL' "$SUPERUSER_EMAIL"
+          test_compose '.services.web.environment.SUPERUSER_USERNAME' "$SUPERUSER_USERNAME"
 
           # Expect docker push args are correct
+          builder="test_builder"
+          progress="test_progress"
+          actual_build_args=$(
+            make docker_build_args \
+              DOCKER_PROGRESS=$progress \
+              DOCKER_BUILDER=$builder
+          )
+          expected_build_args="docker buildx bake web --progress=$progress --builder=$builder"
+
           if [[ ${{ matrix.expected.push }} == "true" ]]; then
-            echo $config | grep -q -- "--push"
-            echo $config | grep -v -q -- "--load"
+            expected_build_args="$expected_build_args --push"
           else
-            echo $config | grep -v -q -- "--push"
-            echo $config | grep -q -- "--load"
+            expected_build_args="$expected_build_args --load"
           fi
 
-          # Expect docker platform is correct
-          echo $config | grep -q -- "platform: linux/amd64"
-          echo $config | grep -q -- "\"platforms\": \[
-            \"linux/amd64\"
-          \]"
+          test_assert "$actual_build_args" "$expected_build_args"
 
+          # Expect docker platform is correct
+          test_compose '.services.web.platform' 'linux/amd64'
+          test_bake '.target.web.platforms[0]' 'linux/amd64'
 
   verify_docker_image:
     runs-on: ubuntu-latest
@@ -62,7 +100,7 @@ jobs:
         id: failure
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
+          version: ${{ steps.build.outputs.version }}
           run: |
             exit 1
         continue-on-error: true
@@ -77,7 +115,7 @@ jobs:
       - name: Check (special characters in command)
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
+          version: ${{ steps.build.outputs.version }}
           run: |
             echo 'this is a question?'
             echo 'a * is born'
@@ -86,8 +124,26 @@ jobs:
       - name: Manage py check
         uses: ./.github/actions/run-docker
         with:
-          image: ${{ steps.build.outputs.tags }}
-          options:
+          version: ${{ steps.build.outputs.version }}
           run: |
-            make update_deps
             make check
+
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - id: build
+        uses: ./.github/actions/build-docker
+
+      - name: Run Test
+        uses: ./.github/actions/run-docker
+        with:
+          version: ${{ steps.build.outputs.version }}
+          services: ''
+          run: |
+            pytest \
+              -n auto \
+              -m 'not es_tests and not needs_locales_compilation and not static_assets and not internal_routes_allowed' \
+              -v src/olympia/amo

.gitignore

@@ -56,7 +56,9 @@ deps/*
 private/
 
 # do not ignore the following files
+!docker-compose._*.yml
 !docker-compose.private.yml
+!docker-compose.production.yml
 !private/README.md
 !deps/.keep