From f5ec13927205e7684b66510770bcc65d757acb20 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Wed, 8 Oct 2025 19:43:29 +0200 Subject: [PATCH 1/3] Decrease docker image size Switch to the *-slim docker image as the base, don't include sandboxes twice in the final image and pass --no-install-recommends to apt. --- Dockerfile | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index d154e5408..91b4acecf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ -FROM python:3.11 AS base +FROM python:3.11-slim AS base ENV PYTHONUNBUFFERED 1 #RUN dpkg --add-architecture i386 RUN apt-get update && \ - apt-get install -y \ + apt-get install --no-install-recommends -y \ git \ libpq-dev \ postgresql-client \ @@ -16,9 +16,13 @@ RUN apt-get update && \ texlive-lang-european \ texlive-lang-czechslovak \ texlive-pstricks \ - ghostscript \ texlive-fonts-recommended \ + tex-gyre \ + ghostscript \ + make \ gcc \ + g++ \ + libc6-dev \ sudo \ libstdc++6 \ zlib1g \ @@ -28,7 +32,8 @@ RUN apt-get update && \ python3-pip \ nodejs \ npm && \ - apt-get clean + apt-get clean && \ + rm -rf /usr/share/doc/texlive* # This is oioioi user linux uid. Setting it is useful in development. # By default we use an unused uid of 1234. @@ -75,7 +80,7 @@ WORKDIR /sio2/deployment RUN mkdir -p /sio2/deployment/logs/{supervisor,runserver} # The stage below is independent of base and can be built in parallel to optimize build time. -FROM python:3.11 AS development-sandboxes +FROM python:3.11-slim AS development-sandboxes ENV DOWNLOAD_DIR=/sio2/sandboxes ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest @@ -84,7 +89,7 @@ ENV MANIFEST_URL=https://downloads.sio2project.mimuw.edu.pl/sandboxes/Manifest ADD $MANIFEST_URL /sio2/Manifest RUN apt-get update && \ - apt-get install -y curl wget bash && \ + apt-get install --no-install-recommends -y curl wget bash && \ apt-get clean COPY download_sandboxes.sh /download_sandboxes.sh @@ -93,14 +98,19 @@ RUN chmod +x /download_sandboxes.sh # Run script to download sandbox data from the given Manifest. RUN ./download_sandboxes.sh -q -y -d $DOWNLOAD_DIR -m $MANIFEST_URL -FROM base AS development - +# For production (or dev with filetracker): Upload sandboxes to built-in filetracker during build +# For dev: The sandboxes will be re-uploaded to s3dedup at runtime via oioioi_init.sh +FROM base AS base_with_sandboxes COPY --from=development-sandboxes /sio2/sandboxes /sio2/sandboxes -RUN chmod +x /sio2/oioioi/download_sandboxes.sh -# For production: Upload sandboxes to built-in filetracker during build -# For dev: These will be re-uploaded to s3dedup at runtime via oioioi_init.sh -RUN ./manage.py supervisor > /dev/null --daemonize --nolaunch=uwsgi && \ +FROM base_with_sandboxes AS base_with_populated_filetracker +RUN ./manage.py supervisor > /dev/null --daemonize \ + --nolaunch={uwsgi,unpackmgr,evalmgr,rankingsd,mailnotifyd,sioworkersd,receive_from_workers} && \ /sio2/oioioi/wait-for-it.sh -t 60 "127.0.0.1:9999" && \ ./manage.py upload_sandboxes_to_filetracker -d /sio2/sandboxes && \ ./manage.py supervisor stop all + +FROM base_with_sandboxes AS development + +FROM base AS development_filetracker +COPY --from=base_with_populated_filetracker /sio2/deployment/media /sio2/deployment/media From 800afb33a3be611156afaf3403c2931afc389ef0 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Mon, 10 Nov 2025 20:38:52 +0100 Subject: [PATCH 2/3] Address a warning for old ENV syntax in Dockerfile --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 91b4acecf..bdc0aa675 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.11-slim AS base -ENV PYTHONUNBUFFERED 1 +ENV PYTHONUNBUFFERED=1 #RUN dpkg --add-architecture i386 RUN apt-get update && \ @@ -55,9 +55,9 @@ RUN sed -i -e "s/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/" /etc/locale.gen && \ # Installing python dependencies USER oioioi -ENV PATH $PATH:/home/oioioi/.local/bin/ +ENV PATH=$PATH:/home/oioioi/.local/bin/ -ENV BERKELEYDB_DIR /usr +ENV BERKELEYDB_DIR=/usr RUN pip3 install --user psycopg2-binary twisted uwsgi RUN pip3 install --user bsddb3==6.2.7 @@ -68,7 +68,7 @@ RUN pip3 install --user -r requirements.txt filetracker[server] RUN pip3 install --user -r requirements_static.txt # Installing node dependencies -ENV PATH $PATH:/sio2/oioioi/node_modules/.bin +ENV PATH=$PATH:/sio2/oioioi/node_modules/.bin RUN npm ci RUN npm run build From 65a4e19ad4be27d5645f903a2b0abbdac444f223 Mon Sep 17 00:00:00 2001 From: Olaf Targowski Date: Thu, 9 Oct 2025 12:45:19 +0200 Subject: [PATCH 3/3] Parallelize uploading sandboxes to filetracker --- .../upload_sandboxes_to_filetracker.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py b/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py index 2e56f8216..bd7f3b11c 100644 --- a/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py +++ b/oioioi/sioworkers/management/commands/upload_sandboxes_to_filetracker.py @@ -1,11 +1,16 @@ -import os -import os.path +from concurrent.futures import ProcessPoolExecutor +from pathlib import Path from django.core.management.base import BaseCommand from oioioi.filetracker.client import get_client +def upload_sandbox(file): + filetracker = get_client() + filetracker.put_file("/sandboxes/" + file.name, str(file)) + + class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( @@ -19,17 +24,12 @@ def add_arguments(self, parser): help = "Upload sandboxes to the Filetracker." - def handle(self, *args, **options): - filetracker = get_client() + def handle(self, *args, **options): print("--- Saving sandboxes to the Filetracker ...", file=self.stdout) - sandboxes_dir = os.fsencode(options["sandboxes_dir"]) - for file in os.listdir(sandboxes_dir): - filename = os.fsdecode(file) - if not filename.endswith(".tar.gz"): - continue - - filetracker.put_file("/sandboxes/" + filename, os.path.join(options["sandboxes_dir"], filename)) + sandboxes_dir = Path(options["sandboxes_dir"]) + with ProcessPoolExecutor() as executor: + executor.map(upload_sandbox, sandboxes_dir.glob('*.tar.gz')) print("--- Done.", file=self.stdout)