Define development and production targets to reduce the need for `mak…

…e update_deps`

.circleci/config.yml

@@ -356,6 +356,9 @@ commands:
       image_tag:
         type: string
         default: "latest"
+      target:
+        type: string
+        default: "production"
     steps:
       - run:
           name: Set environment variables
@@ -364,6 +367,7 @@ commands:
             echo 'export DOCKER_COMMIT=$CIRCLE_SHA1' >> $BASH_ENV
             echo 'export VERSION_BUILD_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV
             echo 'export DOCKER_PUSH=<< parameters.push >>' >> $BASH_ENV
+            echo 'export DOCKER_TARGET=<< parameters.target >>' >> $BASH_ENV
       - run:
           name: Build docker image and push to repo
           command: |

.github/actions/build-docker/action.yml

@@ -17,6 +17,10 @@ inputs:
     required: false
     description: "Docker registry username"
     default: "invalid"
+  target:
+    required: false
+    description: "Build target"
+    default: "production"
 
 outputs:
   version:
@@ -31,11 +35,18 @@ runs:
   steps:
     - name: Validate inputs
       shell: bash
+      id: valid_inputs
       run: |
         if [[ "${{ inputs.push  }}" == "true" && "${{ github.ref }}" == "refs/heads/master" ]]; then
           echo "Cannot push to registry from master branch unless we migrate our master build job to GHA."
           exit 1
         fi
+
+        if [[ "${{ inputs.target }}" != "development" && "${{ inputs.target }}" != "production" ]]; then
+          echo "Invalid target: ${{ inputs.target }}"
+          exit 1
+        fi
+
     # Setup docker to build for multiple architectures
     - name: Set up QEMU
       uses: docker/setup-qemu-action@v3
@@ -89,3 +100,4 @@ runs:
         load: ${{ inputs.push == 'false' }}
       env:
         DOCKER_VERSION: ${{ steps.meta.outputs.version }}
+        DOCKER_TARGET: ${{ inputs.target }}

.github/workflows/build-docker-image.yml

@@ -7,6 +7,14 @@ on:
         description: 'Push the image to registry?'
         default: "false"
         required: false
+      target:
+        description: 'Target stage to build'
+        type: choice
+        options:
+          - "development"
+          - "production"
+        default: "production"
+        required: true
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.inputs.push }}
@@ -26,3 +34,4 @@ jobs:
           push: ${{ inputs.push }}
           username: ${{ secrets.DOCKER_USER }}
           password: ${{ secrets.DOCKER_PASS }}
+          target: ${{ inputs.target }}

.github/workflows/docs.yml

@@ -20,6 +20,8 @@ jobs:
         uses: actions/checkout@v4
       - id: build
         uses: ./.github/actions/build-docker
+        with:
+          target: development
       - name: Setup Pages
         uses: actions/configure-pages@v4
       - name: Build Docs
@@ -28,7 +30,6 @@ jobs:
           image: ${{ steps.build.outputs.tags }}
           options:
           run: |
-            make update_deps
             make docs
       - name: Upload artifact
         uses: actions/upload-pages-artifact@v3

.github/workflows/extract-locales.yml

@@ -23,15 +23,14 @@ jobs:
         id: build
         uses: ./.github/actions/build-docker
         with:
-          load: true
+          target: development
 
       - name: Extract Locales
         uses: ./.github/actions/run-docker
         with:
           image: ${{ steps.build.outputs.tags }}
           options:
           run: |
-            make update_deps
             make extract_locales
 
       - name: Push Locales (dry-run)

.github/workflows/verify-docker-image.yml

@@ -57,6 +57,8 @@ jobs:
       - name: Build Docker image
         id: build
         uses: ./.github/actions/build-docker
+        with:
+          target: development
 
       - name: Create failure
         id: failure
@@ -89,5 +91,4 @@ jobs:
           image: ${{ steps.build.outputs.tags }}
           options:
           run: |
-            make update_deps
             make check

.gitignore

@@ -57,6 +57,7 @@ private/
 
 # do not ignore the following files
 !docker-compose.private.yml
+!docker-compose.extract_deps.yml
 !private/README.md
 !deps/.keep
 

Dockerfile

@@ -105,6 +105,22 @@ RUN \
     # Command to install dependencies
     make -f Makefile-docker update_deps_production
 
+FROM pip_production as pip_development
+
+RUN \
+    # Files needed to run the make command
+    --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
+    # Files required to install pip dependencies
+    --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
+    # Files required to install npm dependencies
+    --mount=type=bind,source=package.json,target=${HOME}/package.json \
+    --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
+    # Mounts for caching dependencies
+    --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
+    --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
+    # Command to install dependencies
+    make -f Makefile-docker update_deps_development
+
 FROM pip_production as locales
 ARG LOCALE_DIR=${HOME}/locale
 # Compile locales
@@ -161,16 +177,25 @@ COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
 # Copy assets from assets
 COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
 
+# We have to reinstall olympia after copying source
+# to ensure the installation syncs files in the src/ directory
+RUN make -f Makefile-docker update_deps_olympia
+
 # version.json is overwritten by CircleCI (see circle.yml).
 # The pipeline v2 standard requires the existence of /app/version.json
 # inside the docker image, thus it's copied there.
 COPY version.json /app/version.json
 
-FROM sources as production
+####################################################################################################
+# There are 2 final stages, development or production. The development stage is used for local
+# and for CI and debugging purposes. The production stage is used for the final image that will
+# be deployed to the production environment. These stages ONLY copy /deps for the final image.
+####################################################################################################
 
+FROM sources as development
 # Copy dependencies from `pip_production`
-COPY --from=pip_production --chown=olympia:olympia /deps /deps
+COPY --from=pip_development --chown=olympia:olympia /deps /deps
 
-# We have to reinstall olympia after copying source
-# to ensure the installation syncs files in the src/ directory
-RUN make -f Makefile-docker update_deps_olympia
+FROM sources as production
+# Copy dependencies from `pip_production`
+COPY --from=pip_production --chown=olympia:olympia /deps /deps

Makefile-os

@@ -8,6 +8,7 @@ DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit")
 VERSION_BUILD_URL ?= build
 # Exporting these variables make them default values for docker-compose*.yml files
 export DOCKER_VERSION ?= local
+export DOCKER_TARGET ?= development
 
 .PHONY: help_redirect
 help_redirect:
@@ -82,21 +83,21 @@ clean_docker: ## Clean up docker containers, images, caches, volumes and local c
 	docker buildx prune -af
 	rm -rf ./deps/**
 
+.PHONY: docker_extract_deps
+docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount
+	rm -rf ./deps/**
+	# Copy the /deps directory from the image to
+	# the ./deps directory on the host.
+	# This ensures the local volume contains the files
+	# from the image and is not replaced by an empty directory
+	# Copying from the image is faster than reinstalling the dependencies
+	docker compose -f docker-compose.extract_deps.yml run --rm --remove-orphans web
+
+.PHONY: up
+up: create_env_file build_docker_image docker_extract_deps docker_compose_up ## Create and start docker compose
+
 .PHONY: initialize_docker
-initialize_docker: create_env_file build_docker_image
-# Run a fresh container from the base image to install deps. Since /deps is
-# shared via a volume in docker-compose.yml, this installs deps for both web
-# and worker containers, and does so without requiring the containers to be up.
-# We just create dummy empty package.json and package-lock.json in deps/ so
-# that docker compose doesn't create dummy ones itself, as they would be owned
-# by root. They don't matter: the ones at the root directory are mounted
-# instead.
-	touch deps/package.json
-	touch deps/package-lock.json
-	# mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container
-	# running `update_deps` will install the dependencies in the /deps directory before running
-	docker compose run --rm web make update_deps
-	docker compose up -d
+initialize_docker: up
 	docker compose exec --user olympia web make initialize
 
 %: ## This directs any other recipe (command) to the web container's make.

docker-compose.extract_deps.yml

@@ -0,0 +1,15 @@
+services:
+  web:
+    extends:
+      file: docker-compose.yml
+      service: web
+    entrypoint:
+      - /bin/bash
+      - -c
+    volumes:
+      - ./deps:/_deps
+      - /deps
+    command:
+      - |
+        chown -R olympia:olympia /_deps
+        cp -r /deps/** /_deps/