Skip to content

Commit

Permalink
TMP: try caching deps dir
Browse files Browse the repository at this point in the history
tmp:fix cache key uniqueness

Make actual package.json available in image

TMP: log errors in cache.

TMP: remove volumes from base

Split mount data olympi

Add compose file var

Use digest instead of version

Better usage of cache action

TMP: fixy

Add base shell possibility

Log contents of site-static
  • Loading branch information
KevinMind committed May 22, 2024
1 parent 2d635c8 commit 45ab1ac
Show file tree
Hide file tree
Showing 18 changed files with 259 additions and 122 deletions.
43 changes: 43 additions & 0 deletions .github/actions/cache-deps/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
name: 'Restore ./deps cache'
description: 'Tries to restore ./deps from cache, installing fresh if missing'
runs:
using: 'composite'
steps:
- name: Keys
id: keys
shell: bash
run: |
echo "key=deps_cache-${{ runner.os }}-${{ github.run_id }}" >> $GITHUB_OUTPUT
echo "pip=pip-${{ hashFiles('requirements/**') }}" >> $GITHUB_OUTPUT
echo "node=node-${{ hashFiles('package*.json') }}" >> $GITHUB_OUTPUT
echo "path=${{ github.workspace }}/deps" >> $GITHUB_OUTPUT
- name: Restore
uses: actions/cache/restore@v3
id: cache
with:
path: ${{ steps.keys.outputs.path }}
key: ${{ steps.keys.outputs.key }}
restore-keys: |
${{ steps.keys.outputs.pip }}
${{ steps.keys.outputs.node }}
- name: Install
shell: bash
env:
DOCKER_SERVICES: web
COMPOSE_FILE: docker-compose.yml
run: |
make up
- name: Save
if: always()
uses: actions/cache/save@v3
with:
path: ${{ steps.keys.outputs.path }}
key: ${{ steps.keys.outputs.key }}

- name: Logs
shell: bash
if: always()
run: docker compose logs
3 changes: 0 additions & 3 deletions .github/actions/run-docker/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,6 @@ runs:
exit 1
fi
# Setup host
make setup
# Start the specified services
make up
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/docs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ jobs:
uses: actions/checkout@v4
- id: build
uses: ./.github/actions/build-docker
- uses: ./.github/actions/cache-deps
- name: Setup Pages
uses: actions/configure-pages@v4
- name: Build Docs
Expand Down
2 changes: 2 additions & 0 deletions .github/workflows/extract-locales.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ jobs:
id: build
uses: ./.github/actions/build-docker

- uses: ./.github/actions/cache-deps

- name: Extract Locales
uses: ./.github/actions/run-docker
with:
Expand Down
57 changes: 48 additions & 9 deletions .github/workflows/verify-docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,25 @@ on:
branches:
- master

concurrency:
group: verify-docker-image
cancel-in-progress: true

jobs:
temporary:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/cache-deps
- uses: ./.github/actions/run-docker
with:
run: |
echo "Hello, World!"
whoami
id
ls -lan /deps
ls -lan /data/olympia
docker_config_check:
runs-on: ubuntu-latest

Expand All @@ -31,6 +49,8 @@ jobs:
id: build
uses: ./.github/actions/build-docker

- uses: ./.github/actions/cache-deps

- name: Create failure
id: failure
uses: ./.github/actions/run-docker
Expand Down Expand Up @@ -75,11 +95,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_addons_versions_files_ratings)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_addons_versions_files_ratings
Expand All @@ -95,11 +117,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_amo_lib_locales_and_signing)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_amo_lib_locales_and_signing
Expand All @@ -115,11 +139,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_needs_locales_compilation)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_needs_locales_compilation
Expand All @@ -135,12 +161,15 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_static_assets)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
ls site-static/js/
make test_static_assets
test_devhub:
Expand All @@ -155,11 +184,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_devhub)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_devhub
Expand All @@ -175,11 +206,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_main)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_main
Expand All @@ -195,11 +228,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_reviewers_and_zadmin)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_reviewers_and_zadmin
Expand All @@ -215,11 +250,13 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_internal_routes_allowed)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_internal_routes_allowed
Expand All @@ -235,10 +272,12 @@ jobs:
with:
target: development

- uses: ./.github/actions/cache-deps

- name: Test (test_es_tests)
uses: ./.github/actions/run-docker
with:
services: ''
version: ${{ steps.build.outputs.version }}
version: ${{ steps.build.outputs.digest }}
run: |
make test_es_tests
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ deps/*
private/

# do not ignore the following files
!docker-compose._*.yml
!docker-compose.build.yml
!docker-compose.mount.yml
!docker-compose.private.yml
!private/README.md
!deps/.keep
Expand Down
13 changes: 6 additions & 7 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -58,9 +58,6 @@ chown olympia:olympia ${HOME}/src/olympia.egg-info
ln -s /deps/bin/uwsgi /usr/bin/uwsgi
ln -s /usr/bin/uwsgi /usr/sbin/uwsgi

# link to the package*.json at ${HOME} so npm can install in /deps
ln -s ${HOME}/package.json /deps/package.json
ln -s ${HOME}/package-lock.json /deps/package-lock.json
EOF

USER olympia:olympia
Expand Down Expand Up @@ -97,8 +94,8 @@ RUN \
# Files required to install pip dependencies
--mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
# Files required to install npm dependencies
--mount=type=bind,source=package.json,target=${HOME}/package.json \
--mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
--mount=type=bind,source=package.json,target=/deps/package.json \
--mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
# Mounts for caching dependencies
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
--mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
Expand All @@ -113,8 +110,8 @@ RUN \
# Files required to install pip dependencies
--mount=type=bind,source=./requirements/dev.txt,target=${HOME}/requirements/dev.txt \
# Files required to install npm dependencies
--mount=type=bind,source=package.json,target=${HOME}/package.json \
--mount=type=bind,source=package-lock.json,target=${HOME}/package-lock.json \
--mount=type=bind,source=package.json,target=/deps/package.json \
--mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
# Mounts for caching dependencies
--mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
--mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
Expand Down Expand Up @@ -181,6 +178,8 @@ COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-stat
# inside the docker image, thus it's copied there.
COPY version.json /app/version.json

COPY package.json package-lock.json ${HOME}/

FROM sources as production

# Copy dependencies from `pip_production`
Expand Down
30 changes: 13 additions & 17 deletions Makefile-os
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
# Variables referenced in docker-compose*.yml should be read from .env, exported and saved in .env
####################################################################################################

export DOCKER_PULL ?= never
DOCKER_BUILDER ?= container
DOCKER_PROGRESS ?= auto
DOCKER_PUSH ?= false
Expand All @@ -17,6 +18,13 @@ override BACKUPS_DIR = $(shell pwd)/backups
override EXPORT_DIR = $(BACKUPS_DIR)/$(shell date +%Y%m%d%H%M%S)
RESTORE_DIR ?= $(BACKUPS_DIR)/$(shell ls -1 backups | sort -r | head -n 1)

DOCKER_COMPOSE_ARGS := -d \
--wait \
--remove-orphans \
--force-recreate \
--quiet-pull \
--build \
--pull=$(DOCKER_PULL) \

.PHONY: help_redirect
help_redirect:
Expand Down Expand Up @@ -123,25 +131,13 @@ clean_docker: docker_compose_down docker_mysqld_volume_remove ## Clean up docker

.PHONY: docker_compose_up
docker_compose_up: docker_mysqld_volume_create ## Start the docker containers
docker compose up $(DOCKER_SERVICES) -d --wait --remove-orphans --force-recreate --quiet-pull $(ARGS)

.PHONY: docker_extract_deps
docker_extract_deps: ## Extract dependencies from the docker image to a local volume mount
# Run a fresh container from the base image to install deps. Since /deps is
# shared via a volume in docker-compose.yml, this installs deps for both web
# and worker containers, and does so without requiring the containers to be up.
# We just create dummy empty package.json and package-lock.json in deps/ so
# that docker compose doesn't create dummy ones itself, as they would be owned
# by root. They don't matter: the ones at the root directory are mounted
# instead.
touch deps/package.json
touch deps/package-lock.json
# mounting ./deps:/deps effectively removes dependencies from the /deps directory in the container
# running `update_deps` will install the dependencies in the /deps directory before running
docker compose run --rm web make update_deps
docker compose up $(DOCKER_SERVICES) $(DOCKER_COMPOSE_ARGS)
# remove the deps container after the other containers are up
docker compose logs deps_container
docker compose rm -f

.PHONY: up
up: setup docker_mysqld_volume_create docker_extract_deps docker_compose_up ## Create and start docker compose
up: setup docker_compose_up ## Create and start docker compose

.PHONY: down
down: docker_compose_down ## Stop the docker containers
Expand Down
24 changes: 24 additions & 0 deletions docker-compose._base.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
networks:
default:

volumes:
deps:
driver: local
driver_opts:
type: none
o: bind
device: ${PWD}/deps
data_redis:
data_elasticsearch:
data_mysqld:
# Keep this value in sync with Makefile-os
# External volumes must be manually created/destroyed
name: addons-server_data_mysqld
external: true
data_rabbitmq:
storage:
driver: local
driver_opts:
type: none
o: bind
device: ${PWD}/storage
17 changes: 17 additions & 0 deletions docker-compose._deps_container.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
services:
# Temporary container to update the dependencies in the deps volume
# before running the web/worker containers.
deps_container:
extends:
file: docker-compose._olympia.yml
service: olympia
# We have to run as olympia to ensure deps has the correct permissions.
entrypoint: ["/bin/sh", "-c"]
user: olympia
command:
- |
whoami
cp /data/olympia/package.json /data/olympia/package-lock.json /deps
ls -lan /deps
make update_deps
ls -lan /deps
Loading

0 comments on commit 45ab1ac

Please sign in to comment.