Rebuild all wheels reproducibly and use wheels from local filesystem

.circleci/config.yml

@@ -138,30 +138,8 @@ common-steps:
         export PKG_PATH=~/packaging/$PKG_NAME/dist/$PKG_NAME-$VERSION_TO_BUILD.tar.gz
         export PKG_VERSION=$VERSION_TO_BUILD
         make $PKG_NAME
-        ls ~/debbuild/packaging/*.deb
+        ls ~/project/build/debbuild/packaging/*.deb
 
-  - &builddebianpackagefromexistingtarball
-    run:
-      name: Build debian package from committed tarball
-      command: |
-        export PKG_PATH=~/project/tarballs/$PKG_NAME-$PKG_VERSION.tar.gz
-
-        # Every tarball should be signed
-        gpg --import ~/project/pubkeys/release_key.pub
-        gpg --verify $PKG_PATH.asc
-
-        # Build debian package
-        make $PKG_NAME
-        export PKG_HASH_1=$(shasum -a 256 ~/debbuild/packaging/$PKG_NAME*.deb | awk '{print $1}')
-        echo $PKG_HASH_1
-
-        # Build debian package again
-        make $PKG_NAME
-        export PKG_HASH_2=$(shasum -a 256 ~/debbuild/packaging/$PKG_NAME*.deb | awk '{print $1}')
-        echo $PKG_HASH_2
-
-        # Fail build if hashes aren’t equal
-        python -c "import os, sys; sys.exit(os.environ['PKG_HASH_1'] != os.environ['PKG_HASH_2'])"
 
   - &addsshkeys
     add_ssh_keys:
@@ -294,6 +272,38 @@ jobs:
             pip install -r test-requirements.txt
             make test
 
+  reprotest-wheels:
+    machine:
+      image: ubuntu-2004:202010-01
+    steps:
+      - checkout
+      - run:
+          name: install test requirements and run tests
+          command: |
+            make install-deps
+            virtualenv -p python3 .venv
+            source .venv/bin/activate
+            pip install -r test-requirements.txt
+            pytest -vvs tests/test_reproducible_wheels.py
+
+  reprotest-debs:
+    docker:
+      - image: circleci/python:3.7-buster
+    steps:
+      - checkout
+      - run:
+          name: install test requirements and run tests
+          command: |
+            make install-deps
+            virtualenv -p python3 .venv
+            source .venv/bin/activate
+            pip install -r test-requirements.txt
+            # Patch reprotest in-place to skip 'setarch' prefix, which fails under containers.
+            # We cannot use Ubuntu 20.04 python3.8 to build Debian 10 python3.7 packages.
+            sudo sed -i -re "292s/^(\s+).*\$/\1return _.prepend_to_build_command_raw('')/" /usr/lib/python3/dist-packages/reprotest/build.py
+            pytest -vvs tests/test_reproducible_debian_packages.py
+
+
   build-buster-securedrop-log:
     docker:
       - image: circleci/python:3.7-buster
@@ -302,7 +312,7 @@ jobs:
       - *removevirtualenv
       - *installdeps
       - *clonesecuredroplog
-      - *getlatestreleasedversion
+      - *getnightlyversion
       - *makesourcetarball
       - *builddebianpackage
 
@@ -329,7 +339,7 @@ jobs:
       - *removevirtualenv
       - *installdeps
       - *clonesecuredropclient
-      - *getlatestreleasedversion
+      - *getnightlyversion
       - *makesourcetarball
       - *builddebianpackage
 
@@ -356,7 +366,7 @@ jobs:
       - *removevirtualenv
       - *installdeps
       - *clonesecuredropproxy
-      - *getlatestreleasedversion
+      - *getnightlyversion
       - *makesourcetarball
       - *builddebianpackage
 
@@ -481,51 +491,12 @@ jobs:
       - *setmetapackageversion
       - *builddebianpackage
 
-  reproducibility-checks:
-    docker:
-      - image: circleci/python:3.7-buster
-    steps:
-      - checkout
-      - *removevirtualenv
-      - *installdeps
-      - run: git lfs pull
-      - run:
-          name: Test build process reproducibility on latest securedrop-client tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-client-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-client' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-      - run:
-          name: Test build process reproducibility on latest securedrop-proxy tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-proxy-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-proxy' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-      - run:
-          name: Test build process reproducibility on latest securedrop-log tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-log-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-log' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-      - run:
-          name: Test build process reproducibility on latest securedrop-export tarball
-          command: |
-            export TARBALL=$(ls ~/project/tarballs/securedrop-export-*.tar.gz)
-            echo ${TARBALL%.tar.gz} | awk -F "-" '{ print $3 }' > ~/sd_version
-            echo 'export PKG_NAME=securedrop-export' >> $BASH_ENV
-            echo 'export PKG_VERSION=$(cat ~/sd_version)' >> $BASH_ENV
-      - *builddebianpackagefromexistingtarball
-
 workflows:
   build-packages:
     jobs:
       - tests
+      - reprotest-wheels
+      - reprotest-debs
       - build-buster-securedrop-client
       - build-buster-securedrop-proxy
       - build-buster-securedrop-workstation-svs-disp
@@ -535,7 +506,6 @@ workflows:
       - build-buster-securedrop-workstation-config
       - build-buster-securedrop-keyring
       - make-dom0-rpm
-      - reproducibility-checks
 
   # Nightly jobs for each package are run in series to ensure there are no
   # conflicts or race conditions when committing deb packages to git-lfs.

.gitignore

@@ -1,3 +1,4 @@
 tests/__pycache__/
 debhelper-build-stamp
 *.debhelper.log
+build/

Makefile

@@ -2,11 +2,11 @@ DEFAULT_GOAL: help
 
 .PHONY: securedrop-proxy
 securedrop-proxy: ## Builds Debian package for securedrop-proxy code
-	PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
+	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
 
 .PHONY: securedrop-client
 securedrop-client: ## Builds Debian package for securedrop-client code
-	PKG_NAME="securedrop-client" ./scripts/build-debianpackage
+	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-client" ./scripts/build-debianpackage
 
 .PHONY: securedrop-workstation-config
 securedrop-workstation-config: ## Builds Debian metapackage for Qubes Workstation base dependencies
@@ -26,11 +26,11 @@ securedrop-workstation-viewer: ## Builds Debian metapackage for Disposable VM de
 
 .PHONY: securedrop-export
 securedrop-export: ## Builds Debian package for Qubes Workstation export scripts
-	PKG_NAME="securedrop-export" ./scripts/build-debianpackage
+	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-export" ./scripts/build-debianpackage
 
 .PHONY: securedrop-log
 securedrop-log: ## Builds Debian package for Qubes Workstation securedrop-log scripts
-	PKG_NAME="securedrop-log" ./scripts/build-debianpackage
+	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-log" ./scripts/build-debianpackage
 
 .PHONY: securedrop-keyring
 securedrop-keyring: ## Builds Debian package containing the release key
@@ -54,16 +54,16 @@ build-wheels: ## Builds the wheels and adds them to the localwheels directory
 	@printf "to push these changes to the FPF PyPI index\n"
 
 .PHONY: test
-test: ## Run test suite
-	pytest -v tests/
+test: ## Run simple test suite (skips reproducibility checks)
+	pytest -v tests/test_update_requirements.py
 
 .PHONY: clean
 clean: ## Removes all non-version controlled packaging artifacts
 	rm -rf localwheels/*
 
 .PHONY: reprotest
-reprotest: ## Reproducibility test, currently only for wheels
-	pytest -vvs tests/test_reproducible_wheels.py
+reprotest: ## Runs only reproducibility tests, for .deb and .whl files
+	pytest -vvs tests/test_reproducible_*.py
 
 .PHONY: help
 help: ## Prints this message and exits

README.md

@@ -34,8 +34,8 @@ Maintainers of `securedrop-client` and `securedrop-proxy` must ensure that
 the requirements files which are used for build of these packages (`build-requirements.txt`)
 using `make requirements` are kept up to date in latest `main` of those repositories.
 
-If new dependencies were added in the `requirements.txt` of that
-repo that are not in the FPF PyPI mirror (https://pypi.securedrop.org/), then the maintainer needs
+If new dependencies were added in the `build-requirements.txt` of that
+repo that are not in the FPF PyPI mirror (`./localwheels/` in this repository), then the maintainer needs
 to do the following (we are taking `securedrop-client` project as example):
 
 ### 0. Create updated build-requirements.txt for the project

scripts/build-debianpackage

@@ -23,12 +23,12 @@ export DH_PIP_EXTRA_ARGS="--no-cache-dir --require-hashes"
 
 # Declare general packaging building workspace; subdirs will
 # be created within, to build specific packages.
-TOP_BUILDDIR="$HOME/debbuild/packaging"
+TOP_BUILDDIR="$PWD/build/debbuild/packaging"
 mkdir -p "$TOP_BUILDDIR"
 rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}"
 mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}"
 # Move changelog into place (we have separate changelogs for each platform)
-PLATFORM="$(lsb_release -sc)"
+PLATFORM="${PKG_PLATFORM:-buster}"
 
 # Validate required args.
 if [[ -z "${PKG_NAME:-}" ]]; then
@@ -83,11 +83,16 @@ function build_source_tarball() {
     rm -rf "$build_dir"
     git clone "$repo_url" "$build_dir"
 
-    # Verify tag, using only the prod key
-    verify_git_tag "$build_dir" "$PKG_VERSION"
+    if [[ -n "$PKG_GITREF" ]]; then
+        # Can't expect a prod sig on a gitref, likely a feature branch
+        git -C "$build_dir" checkout "$PKG_GITREF"
+    else
+        # Verify tag, using only the prod key
+        verify_git_tag "$build_dir" "$PKG_VERSION"
+        # Tag is verified, proceed with checkout
+        git -C "$build_dir" checkout "$PKG_VERSION"
+    fi
 
-    # Tag is verified, proceed with checkout
-    git -C "$build_dir" checkout "$PKG_VERSION"
     (cd "$build_dir" && LC_ALL="C.UTF-8" python setup.py sdist)
 
     # Initial tarball will contain timestamps from NOW, let's repack

scripts/sync-sha256sums

@@ -9,5 +9,5 @@ set -o pipefail
 cd ./localwheels/
 sha256sum * > ../sha256sums.txt
 
-printf "Now you must sign the generated sha256sums.txt file."
-gpg --armor --output ../sha256sums.txt.asc --detach-sig  ../sha256sums.txt
+echo "Now you must sign the generated sha256sums.txt file:"
+echo "gpg --armor --output ../sha256sums.txt.asc --detach-sig  ../sha256sums.txt"

scripts/update-requirements

@@ -11,7 +11,7 @@ from pprint import pprint
 
 
 def main():
-    PKG_DIR = os.environ["PKG_DIR"]
+    PKG_DIR = os.environ.get("PKG_DIR", "")
 
     if not PKG_DIR:
         print("Set PKG_DIR of the project")