diff --git a/.circleci/config.yml b/.circleci/config.yml index 830dfca16a4..cdd989538f6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -356,6 +356,9 @@ commands: image_tag: type: string default: "latest" + target: + type: string + default: "production" steps: - run: name: Set environment variables @@ -364,6 +367,7 @@ commands: echo 'export DOCKER_COMMIT=$CIRCLE_SHA1' >> $BASH_ENV echo 'export VERSION_BUILD_URL=$CIRCLE_BUILD_URL' >> $BASH_ENV echo 'export DOCKER_PUSH=<< parameters.push >>' >> $BASH_ENV + echo 'export DOCKER_TARGET=<< parameters.target >>' >> $BASH_ENV - run: name: Build docker image and push to repo command: | diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index b709f7f51a2..75d1f859988 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -17,6 +17,10 @@ inputs: required: false description: "Docker registry username" default: "invalid" + target: + required: false + description: "Build target" + default: "production" outputs: version: @@ -31,11 +35,18 @@ runs: steps: - name: Validate inputs shell: bash + id: valid_inputs run: | if [[ "${{ inputs.push }}" == "true" && "${{ github.ref }}" == "refs/heads/master" ]]; then echo "Cannot push to registry from master branch unless we migrate our master build job to GHA." exit 1 fi + + if [[ "${{ inputs.target }}" != "development" && "${{ inputs.target }}" != "production" ]]; then + echo "Invalid target: ${{ inputs.target }}" + exit 1 + fi + # Setup docker to build for multiple architectures - name: Set up QEMU uses: docker/setup-qemu-action@v3 @@ -89,3 +100,4 @@ runs: load: ${{ inputs.push == 'false' }} env: DOCKER_VERSION: ${{ steps.meta.outputs.version }} + DOCKER_TARGET: ${{ inputs.target }} diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 02b36465318..5d2a796b06e 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -7,6 +7,14 @@ on: description: 'Push the image to registry?' default: "false" required: false + target: + description: 'Target stage to build' + type: choice + options: + - "development" + - "production" + default: "production" + required: true concurrency: group: ${{ github.workflow }}-${{ github.event.inputs.push }} @@ -26,3 +34,4 @@ jobs: push: ${{ inputs.push }} username: ${{ secrets.DOCKER_USER }} password: ${{ secrets.DOCKER_PASS }} + target: ${{ inputs.target }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 712b76430b6..0ee8ca93b48 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -20,6 +20,8 @@ jobs: uses: actions/checkout@v4 - id: build uses: ./.github/actions/build-docker + with: + target: development - name: Setup Pages uses: actions/configure-pages@v4 - name: Build Docs @@ -28,7 +30,6 @@ jobs: image: ${{ steps.build.outputs.tags }} options: run: | - make update_deps make docs - name: Upload artifact uses: actions/upload-pages-artifact@v3 diff --git a/.github/workflows/extract-locales.yml b/.github/workflows/extract-locales.yml index ad5edba4d0c..7307b716616 100644 --- a/.github/workflows/extract-locales.yml +++ b/.github/workflows/extract-locales.yml @@ -23,7 +23,7 @@ jobs: id: build uses: ./.github/actions/build-docker with: - load: true + target: development - name: Extract Locales uses: ./.github/actions/run-docker @@ -31,7 +31,6 @@ jobs: image: ${{ steps.build.outputs.tags }} options: run: | - make update_deps make extract_locales - name: Push Locales (dry-run) diff --git a/.github/workflows/verify-docker-image.yml b/.github/workflows/verify-docker-image.yml index a27f7bbdbe1..d50f29a7161 100644 --- a/.github/workflows/verify-docker-image.yml +++ b/.github/workflows/verify-docker-image.yml @@ -57,6 +57,8 @@ jobs: - name: Build Docker image id: build uses: ./.github/actions/build-docker + with: + target: development - name: Create failure id: failure @@ -89,5 +91,4 @@ jobs: image: ${{ steps.build.outputs.tags }} options: run: | - make update_deps make check diff --git a/.gitignore b/.gitignore index 02f32301335..12d8042681a 100644 --- a/.gitignore +++ b/.gitignore @@ -57,6 +57,7 @@ private/ # do not ignore the following files !docker-compose.private.yml +!docker-compose.extract_deps.yml !private/README.md !deps/.keep diff --git a/Dockerfile b/Dockerfile index 0cea0eeed1f..68776483284 100644 --- a/Dockerfile +++ b/Dockerfile @@ -105,6 +105,22 @@ RUN \ # Command to install dependencies make -f Makefile-docker update_deps_production +FROM pip_production as pip_development + +RUN \ + # Files needed to run the make command + --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \ + # Files required to install pip dependencies + --mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \ + # Files required to install npm dependencies + --mount=type=bind,source=package.json,target=${HOME}/package.json \ + --mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \ + # Mounts for caching dependencies + --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \ + --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \ + # Command to install dependencies + make -f Makefile-docker update_deps_development + FROM pip_production as locales ARG LOCALE_DIR=${HOME}/locale # Compile locales @@ -161,16 +177,25 @@ COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale # Copy assets from assets COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static +# We have to reinstall olympia after copying source +# to ensure the installation syncs files in the src/ directory +RUN make -f Makefile-docker update_deps_olympia + # version.json is overwritten by CircleCI (see circle.yml). # The pipeline v2 standard requires the existence of /app/version.json # inside the docker image, thus it's copied there. COPY version.json /app/version.json -FROM sources as production +#################################################################################################### +# There are 2 final stages, development or production. The development stage is used for local +# and for CI and debugging purposes. The production stage is used for the final image that will +# be deployed to the production environment. These stages ONLY copy /deps for the final image. +#################################################################################################### +FROM sources as development # Copy dependencies from `pip_production` -COPY --from=pip_production --chown=olympia:olympia /deps /deps +COPY --from=pip_development --chown=olympia:olympia /deps /deps -# We have to reinstall olympia after copying source -# to ensure the installation syncs files in the src/ directory -RUN make -f Makefile-docker update_deps_olympia +FROM sources as production +# Copy dependencies from `pip_production` +COPY --from=pip_production --chown=olympia:olympia /deps /deps diff --git a/Makefile-os b/Makefile-os index c06f3bcd27a..5f1beff4798 100644 --- a/Makefile-os +++ b/Makefile-os @@ -8,6 +8,7 @@ DOCKER_COMMIT ?= $(shell git rev-parse HEAD || echo "commit") VERSION_BUILD_URL ?= build # Exporting these variables make them default values for docker-compose*.yml files export DOCKER_VERSION ?= local +export DOCKER_TARGET ?= development .PHONY: help_redirect help_redirect: @@ -82,21 +83,21 @@ clean_docker: ## Clean up docker containers, images, caches, volumes and local c docker buildx prune -af rm -rf ./deps/** +.PHONY: docker_extract_deps +docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount + rm -rf ./deps/** + # Copy the /deps directory from the image to + # the ./deps directory on the host. + # This ensures the local volume contains the files + # from the image and is not replaced by an empty directory + # Copying from the image is faster than reinstalling the dependencies + docker compose -f docker-compose.extract_deps.yml run --rm --remove-orphans web + +.PHONY: up +up: create_env_file build_docker_image docker_extract_deps docker_compose_up ## Create and start docker compose + .PHONY: initialize_docker -initialize_docker: create_env_file build_docker_image -# Run a fresh container from the base image to install deps. Since /deps is -# shared via a volume in docker-compose.yml, this installs deps for both web -# and worker containers, and does so without requiring the containers to be up. -# We just create dummy empty package.json and package-lock.json in deps/ so -# that docker compose doesn't create dummy ones itself, as they would be owned -# by root. They don't matter: the ones at the root directory are mounted -# instead. - touch deps/package.json - touch deps/package-lock.json - # mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container - # running `update_deps` will install the dependencies in the /deps directory before running - docker compose run --rm web make update_deps - docker compose up -d +initialize_docker: up docker compose exec --user olympia web make initialize %: ## This directs any other recipe (command) to the web container's make. diff --git a/docker-compose.extract_deps.yml b/docker-compose.extract_deps.yml new file mode 100644 index 00000000000..4ad5e1a12e4 --- /dev/null +++ b/docker-compose.extract_deps.yml @@ -0,0 +1,15 @@ +services: + web: + extends: + file: docker-compose.yml + service: web + entrypoint: + - /bin/bash + - -c + volumes: + - ./deps:/_deps + - /deps + command: + - | + chown -R olympia:olympia /_deps + cp -r /deps/** /_deps/