Merge pull request #14085 from romeroalx/pip-doc-hashes

GH actions: check that only pinned pip packages are pulled when building docs
PowerDNS · May 2, 2024 · 18f2833 · 18f2833
2 parents 5e38664 + 6e80439
commit 18f2833
Show file tree

Hide file tree

Showing 8 changed files with 636 additions and 24 deletions.
diff --git a/.github/actions/spell-check/allow.txt b/.github/actions/spell-check/allow.txt
@@ -2547,6 +2547,7 @@ pbegin
 pbf
 PBKDF
 PBpq
+pbr
 pbtag
 pcall
 PCDNSSEC
@@ -3144,6 +3145,7 @@ sched
 schemaversion
 scl
 sclo
+scm
 SCNu
 SCombo
 SConnect

diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml
@@ -14,9 +14,27 @@ jobs:
   build-upload-docs:
     name: Build and upload docs
     runs-on: ubuntu-22.04
+    env:
+      SERVICE_IP_ADDR: 127.0.0.1
+    services:
+      database:
+        image: epicwink/proxpi
+        ports:
+          - 5000:5000
+        options: >-
+          --restart always
+    outputs:
+      pip-list-auth: ${{ steps.pip-logs-auth.outputs.list-auth }}
+      pip-list-rec: ${{ steps.pip-logs-rec.outputs.list-rec }}
+      pip-list-dnsdist: ${{ steps.pip-logs-dnsdist.outputs.list-dnsdist }}
     steps:
       - uses: PowerDNS/pdns/set-ubuntu-mirror@meta
       - uses: actions/checkout@v4
+      # Configure pip index-url set to proxpi
+      - run: pip config set global.index-url http://${{ env.SERVICE_IP_ADDR }}:5000/index/
+      - run: pip config set global.trusted-host ${{ env.SERVICE_IP_ADDR }}
+      - id: proxpi-docker
+        run: echo "id=$(docker ps | grep "epicwink/proxpi" | awk '{print $1}')" >> "$GITHUB_OUTPUT"
       - run: build-scripts/gh-actions-setup-inv-no-dist-upgrade  # this runs apt update
       - run: inv install-doc-deps
       - run: inv install-doc-deps-pdf
@@ -36,6 +54,7 @@ jobs:
 
       # Auth
       - run: inv ci-docs-build
+
       - run: mv html auth-html-docs
         working-directory: ./docs/_build
       - run: tar cf auth-html-docs.tar auth-html-docs
@@ -57,6 +76,10 @@ jobs:
           DOCS_HOST: ${{vars.DOCS_HOST}}
           AUTH_DOCS_DIR: ${{vars.AUTH_DOCS_DIR}}
         if: ${{github.ref_name == 'master' && steps.setup-ssh.outputs.have_ssh_key != ''}}
+      - run: docker logs ${{ steps.proxpi-docker.outputs.id }} 2>&1 | grep whl | awk '{print $8}' | cut -d "/" -f 4 | awk -F'-' '{print $1"=="$2}' | sort -u --ignore-case > /tmp/proxpi-auth.log
+      - id: pip-logs-auth
+        run: echo "list-auth=$(cat /tmp/proxpi-auth.log | base64 -w0)" >> "$GITHUB_OUTPUT"
+      - run: sudo sh -c "truncate -s 0 /var/lib/docker/containers/${{ steps.proxpi-docker.outputs.id }}*/${{ steps.proxpi-docker.outputs.id }}*-json.log"
 
       # Rec
       - run: inv ci-docs-rec-generate
@@ -86,6 +109,10 @@ jobs:
           REC_DOCS_DIR: ${{vars.REC_DOCS_DIR}}
         if: ${{github.ref_name == 'master' && steps.setup-ssh.outputs.have_ssh_key != ''}}
         working-directory: ./pdns/recursordist
+      - run: docker logs ${{ steps.proxpi-docker.outputs.id }} 2>&1 | grep whl | awk '{print $8}' | cut -d "/" -f 4 | awk -F'-' '{print $1"=="$2}' | sort -u --ignore-case > /tmp/proxpi-rec.log
+      - id: pip-logs-rec
+        run: echo "list-rec=$(cat /tmp/proxpi-rec.log | base64 -w0)" >> "$GITHUB_OUTPUT"
+      - run: sudo sh -c "truncate -s 0 /var/lib/docker/containers/${{ steps.proxpi-docker.outputs.id }}*/${{ steps.proxpi-docker.outputs.id }}*-json.log"
 
       # DNSdist
       - run: inv ci-docs-build
@@ -112,3 +139,18 @@ jobs:
           DOCS_HOST: ${{vars.DOCS_HOST}}
         if: ${{github.ref_name == 'master' && steps.setup-ssh.outputs.have_ssh_key != ''}}
         working-directory: ./pdns/dnsdistdist
+      - run: docker logs ${{ steps.proxpi-docker.outputs.id }} 2>&1 | grep whl | awk '{print $8}' | cut -d "/" -f 4 | awk -F'-' '{print $1"=="$2}' | sort -u --ignore-case > /tmp/proxpi-dnsdist.log
+      - id: pip-logs-dnsdist
+        run: echo "list-dnsdist=$(cat /tmp/proxpi-dnsdist.log | base64 -w0)" >> "$GITHUB_OUTPUT"
+
+  validate-pip-hashes:
+    name: Validate list of packages and hashes
+    runs-on: ubuntu-22.04
+    needs: build-upload-docs
+    steps:
+      - uses: actions/checkout@v4
+      - run: for i in `echo "${{ needs.build-upload-docs.outputs.pip-list-auth }}" | base64 -d | sed 's/_/-/' | egrep -v "pip==|setuptools==|wheel==|setuptools-git=="`; do grep -qq -i $i docs/requirements.txt || ( echo "$i not found" && exit 1 ); done
+      - run: for i in `echo "${{ needs.build-upload-docs.outputs.pip-list-rec }}" | base64 -d | sed 's/_/-/' | egrep -v "pip==|setuptools==|wheel==|setuptools-git=="`; do grep -qq -i $i docs/requirements.txt || ( echo "$i not found" && exit 1 ); done
+        working-directory: ./pdns/recursordist
+      - run: for i in `echo "${{ needs.build-upload-docs.outputs.pip-list-dnsdist }}" | base64 -d | sed 's/_/-/' | egrep -v "pip==|setuptools==|wheel==|setuptools-git=="`; do grep -qq -i $i docs/requirements.txt || ( echo "$i not found" && exit 1 ); done
+        working-directory: ./pdns/dnsdistdist
diff --git a/docs/requirements.in b/docs/requirements.in
@@ -10,3 +10,6 @@ guzzle_sphinx_theme
 docutils!=0.15,<0.18
 jinja2<3.1.0
 pyyaml==6.0.1
+packaging==24.0
+setuptools-scm==8.0.3 # setup-requires for sphinxcontrib-openapi
+pbr # setup-requires for sphinxcontrib-fulltoc
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --generate-hashes requirements.in
@@ -184,10 +184,17 @@ markupsafe==2.1.3 \
     --hash=sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc \
     --hash=sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2
     # via jinja2
-packaging==23.1 \
-    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
-    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
-    # via sphinx
+packaging==24.0 \
+    --hash=sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5 \
+    --hash=sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9
+    # via
+    #   -r requirements.in
+    #   setuptools-scm
+    #   sphinx
+pbr==6.0.0 \
+    --hash=sha256:4a7317d5e3b17a3dccb6a8cfe67dab65b20551404c52c8ed41279fa4f0cb4cda \
+    --hash=sha256:d1377122a5a00e2f940ee482999518efe16d745d423a670c27773dfbc3c9a7d9
+    # via -r requirements.in
 pygments==2.15.1 \
     --hash=sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c \
     --hash=sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1
@@ -273,6 +280,10 @@ requests==2.31.0 \
     --hash=sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f \
     --hash=sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1
     # via sphinx
+setuptools-scm==8.0.3 \
+    --hash=sha256:0169fd70197efda2f8c4d0b2a7a3d614431b488116f37b79d031e9e7ec884d8c \
+    --hash=sha256:813822234453438a13c78d05c8af29918fbc06f88efb33d38f065340bbb48c39
+    # via -r requirements.in
 six==1.16.0 \
     --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
     --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
@@ -316,6 +327,14 @@ sphinxcontrib-websupport==1.2.4 \
     --hash=sha256:4edf0223a0685a7c485ae5a156b6f529ba1ee481a1417817935b20bde1956232 \
     --hash=sha256:6fc9287dfc823fe9aa432463edd6cea47fa9ebbf488d7f289b322ffcfca075c7
     # via sphinx
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via setuptools-scm
+typing-extensions==4.11.0 \
+    --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \
+    --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a
+    # via setuptools-scm
 urllib3==2.0.7 \
     --hash=sha256:c97dfde1f7bd43a71c8d2a58e369e9b2bf692d1334ea9f9cae55add7d0dd0f84 \
     --hash=sha256:fdb6d215c776278489906c2f8916e6e7d4f5a9b602ccbcfdf7f016fc8da0596e

diff --git a/pdns/dnsdistdist/docs/requirements.in b/pdns/dnsdistdist/docs/requirements.in
@@ -0,0 +1,12 @@
+# To generate requirements.txt, install pip-tools and run:
+#  pip-compile --generate-hashes requirements.in
+
+Sphinx>=1.5.0,!=1.8.0,<2.0
+https://github.com/PowerDNS/sphinx-lua/archive/refs/heads/pdns.zip
+https://github.com/PowerDNS/sphinx-jsondomain/archive/refs/heads/no-type-links.zip
+changelog>=0.5.6,<0.6
+sphinxcontrib-httpdomain
+sphinxcontrib-fulltoc
+docutils!=0.15,<0.18
+jinja2<3.1.0
+alabaster==0.7.13