lets just try dockah

.github/workflows/ci.yml

@@ -9,60 +9,102 @@ on:
   workflow_dispatch:  # generally only for the "combine-prs" workflow
 permissions:
   contents: read
+  packages: read
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
   cancel-in-progress: true
 jobs:
-  deps:
+  build:
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-      - name: Install platform dependencies
-        run: |
-          sudo apt -y update
-          sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v5
+      - name: Expose GitHub Runtime
+        uses: crazy-max/ghaction-github-runtime@v3
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Cache RUN --mount=type=cache /var/cache/apt
+        uses: actions/cache@v3
         with:
-          python-version-file: '.python-version'
-      - name: pip cache
-        uses: actions/cache@v4
+          path: var-cache-apt
+          key: var-cache-apt-${{ hashFiles('Dockerfile') }}
+          restore-keys: |
+            var-cache-apt-
+      - name: Cache RUN --mount=type=cache /var/lib/apt
+        uses: actions/cache@v3
         with:
-          path: ~/.cache/pip
-          key: pip-${{ runner.os }}
-      - name: Cache built Python environment
-        uses: actions/cache@v4
+          path: var-lib-apt
+          key: var-lib-apt-${{ hashFiles('Dockerfile') }}
+          restore-keys: |
+            var-lib-apt-
+      - name: Cache RUN --mount=type=cache /root/.cache/pip
+        uses: actions/cache@v3
+        with:
+          path: root-cache-pip
+          key: root-cache-pip-${{ hashFiles('requirements/*.txt') }}
+          restore-keys: |
+            root-cache-pip-
+      - name: Cache RUN --mount=type=cache /root/.npm
+        uses: actions/cache@v3
+        with:
+          path: root-npm
+          key: root-npm-${{ hashFiles('package.json', 'package-lock.json') }}
+          restore-keys: |
+            root-npm-
+      - name: inject var-cache-apt into docker
+        uses: reproducible-containers/buildkit-cache-dance@v2.1.4
         with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
-      - name: Install Python dependencies
+          cache-source: var-cache-apt
+          cache-target: /var/cache/apt
+          skip-extraction: ${{ steps.cache-var-cache-apt.outputs.cache-hit }}
+      - name: inject var-lib-apt into docker
+        uses: reproducible-containers/buildkit-cache-dance@v2.1.4
+        with:
+          cache-source: var-lib-apt
+          cache-target: /var/lib/apt
+          skip-extraction: ${{ steps.cache-var-lib-apt.outputs.cache-hit }}
+      - name: inject root-cache-pip into docker
+        uses: reproducible-containers/buildkit-cache-dance@v2.1.4
+        with:
+          cache-source: root-cache-pip
+          cache-target: /root/.cache/pip
+          skip-extraction: ${{ steps.cache-root-cache-pip.outputs.cache-hit }}
+      - name: inject root-npm into docker
+        uses: reproducible-containers/buildkit-cache-dance@v2.1.4
+        with:
+          cache-source: root-npm
+          cache-target: /root/.npm
+          skip-extraction: ${{ steps.cache-root-npm.outputs.cache-hit }}
+      - name: Build Docker Image
         run: |
-          pip install -U setuptools wheel pip
-          pip install -r requirements.txt --no-deps
-          pip install -r requirements/dev.txt
-          pip check
+            echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u ${{ github.repository_owner }} --password-stdin
+            docker buildx build --build-arg="DEVEL=yes" --build-arg="CI=yes" --cache-to type=gha,mode=max --cache-from type=gha --push -t ghcr.io/pypi/warehouse:ci-${{ github.run_id }} .
   test:
-    # Time out if our test suite has gotten hung
-    timeout-minutes: 15
-    needs: deps
+    needs: build
     strategy:
       matrix:
         include:
           - name: Tests
-            command: bin/tests --postgresql-host localhost
+            command: bin/tests --postgresql-host postgres
           - name: Lint
             command: bin/lint
-          - name: User Documentation
-            command: bin/user-docs
-          - name: Developer Documentation
-            command: bin/dev-docs
           - name: Dependencies
-            command: bin/github-actions-deps
+            command: bin/deps
           - name: Licenses
             command: bin/licenses
           - name: Translations
             command: bin/translations
+          - name: User Documentation
+            command: bin/user-docs
+          - name: Developer Documentation
+            command: bin/dev-docs
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/pypi/warehouse:ci-${{ github.run_id }}
+      env:
+        BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://stripe:12111 api_version=2020-08-27
     services:
       postgres:
         image: ${{ (matrix.name == 'Tests') && 'postgres:14.11' || '' }}
@@ -77,31 +119,15 @@ jobs:
         ports:
           - 12111:12111
     name: ${{ matrix.name }}
-    env:
-      BILLING_BACKEND: warehouse.subscriptions.services.MockStripeBillingService api_base=http://localhost:12111 api_version=2020-08-27
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
-      - name: Install platform dependencies
-        run: |
-          sudo apt -y update
-          sudo apt -y install libcurl4-openssl-dev libssl-dev pkg-config libxml2-dev libxslt-dev
-      - uses: actions/setup-python@v5
-        with:
-          python-version-file: '.python-version'
       - name: Cache mypy results
         if: ${{ (matrix.name == 'Lint') }}
         uses: actions/cache@v4
         with:
           path: |
               dev/.mypy_cache
-          key: ${{ runner.os }}-mypy-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
-      - name: Restore built Python environment from deps
-        uses: actions/cache/restore@v4
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ runner.os }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
-          # Since we don't install deps again, we fail if we can't restore the cache (timeout, etc)
-          fail-on-cache-miss: true
+          key: ${{ runner.os }}-mypy-${{ hashFiles('requirements.txt', 'requirements/*.txt') }}
       - name: Run ${{ matrix.name }}
         run: ${{ matrix.command }}

Dockerfile

@@ -112,6 +112,10 @@ FROM python:3.11.8-slim-bookworm as build
 # test dependencies.
 ARG DEVEL=no
 
+# Define whether we're building a CI image. This will include all the docs stuff
+# as well for the matrix!
+ARG CI=no
+
 # To enable Ipython in the development environment set to yes (for using ipython
 # as the warehouse shell interpreter,
 # i.e. 'docker compose run --rm web python -m warehouse shell --type=ipython')
@@ -176,6 +180,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
                     -r /tmp/requirements/deploy.txt \
                     -r /tmp/requirements/main.txt \
                     $(if [ "$DEVEL" = "yes" ]; then echo '-r /tmp/requirements/tests.txt -r /tmp/requirements/lint.txt'; fi) \
+                    $(if [ "$CI" = "yes" ]; then echo '-r /tmp/requirements/docs-dev.txt -r /tmp/requirements/docs-user.txt -r /tmp/requirements/docs-blog.txt'; fi ) \
     && pip check \
     && find /opt/warehouse -name '*.pyc' -delete
 
@@ -198,6 +203,10 @@ WORKDIR /opt/warehouse/src/
 # test dependencies.
 ARG DEVEL=no
 
+# Define whether we're building a CI image. This will include all the docs stuff
+# as well for the matrix!
+ARG CI=no
+
 # This is a work around because otherwise postgresql-client bombs out trying
 # to create symlinks to these directories.
 RUN set -x \
@@ -211,8 +220,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     set -x \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
-        libpq5 libxml2 libxslt1.1 libcurl4  \
+        libpq5 libxml2 libxslt1.1 libcurl4 \
         $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev vim'; fi) \
+        $(if [ "$CI" = "yes" ]; then echo 'git'; fi) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 

requirements.txt

@@ -1,4 +1,4 @@
-# This is only used for dependabot and Github Actions CI
+# This is only used for dependabot
 -r requirements/main.txt
 -r requirements/deploy.txt
 -r requirements/docs-dev.txt

warehouse/forklift/legacy.py

@@ -440,7 +440,7 @@ def process_formdata(self, valuelist):
 
 # TODO: Eventually this whole validation thing should move to the packaging
 #       library and we should just call that. However until PEP 426 is done
-#       that library won't have an API for this.
+#       that library won't have an API for this. ref: #14718
 class MetadataForm(forms.Form):
     # Metadata version
     metadata_version = wtforms.StringField(

warehouse/locale/Makefile

@@ -39,7 +39,7 @@ build-mos: compile-pot
 translations: compile-pot
 ifneq ($(GITHUB_ACTIONS), false)
 	cd ../../; \
-	git diff --quiet ./warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
+	cmp -s ./warehouse/locale/messages.pot /opt/warehouse/src/warehouse/locale/messages.pot || (echo "There are outstanding translations, run 'make translations' and commit the changes."; exit 1)
 else
 endif