Reorganize how wheels are stored, stop special-casing bootstrap

Each project, whether it's securedrop-client or the
workstation-bootstrap, will now store wheels and associated metadata in
the exact same way. Moving files around will happen in follow-up
commits.

securedrop-client/
  debian/
  sha256sums.txt
  sha256sums.txt.asc
  wheels/

The workstation-bootstrap is similar, except it also contains its
requirements files.

workstation-bootstrap/
  build-requirements.txt
  requirements.in
  requirements.txt
  sha256sums.txt
  sha256sums.txt.asc
  wheels/

The main goal of this refactor is to make room for the new
securedrop-app-code bootstrap and wheels. The main scripts now take a
`--project` parameter, which is the path in this repository, and
`--pkg-dir`, which is the path to the Git checkout of it (previously a
PKG_DIR environment variable). In nearly all cases backwards-compat code
has been added so it should do the right thing based on old
documentation.

Makefile:
* Updates for how scripts are now invoked.
* Drop misleading "clean" target, people can use git-clean(1) directly.

scripts/build-debianpackage:
* Validate $PKG_NAME before we use it.
* Set $WHEELS_DIR to that package's wheel directory.
* Only verify sha256sums.txt if it exists (securedrop-export and
  metapackages have no Python dependencies).

scripts/build-sync-wheels:
* Switch to --pkg-dir/--project args, with backwards-compat.
* Allow specifying where requirements.txt lives.
* Drop "cache" terminology since this is persistent storage.
* Remove dead commented-out code.

scripts/install-deps:
* Debian 11 is our baseline now, fix typo.
* Look for and install wheels from workstation-bootstrap now.

scripts/sync-sha256sums:
* Require a directory instead of looking for a BOOTSTRAP variable.

scripts/update-requirements:
* Switch to --pkg-dir/--project args, with backwards-compat.
* Allow specifying where requirements.txt lives.
* Use pathlib internally

scripts/verify-sha256sum-signature:
* Require a directory instead of looking for a BOOTSTRAP variable.

Refs <freedomofpress/securedrop#5901>.

Makefile

@@ -1,12 +1,13 @@
 DEFAULT_GOAL: help
+SHELL := /bin/bash
 
 .PHONY: securedrop-proxy
 securedrop-proxy: ## Builds Debian package for securedrop-proxy code
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-proxy" ./scripts/build-debianpackage
 
 .PHONY: securedrop-client
 securedrop-client: ## Builds Debian package for securedrop-client code
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-client" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-client" ./scripts/build-debianpackage
 
 .PHONY: securedrop-workstation-config
 securedrop-workstation-config: ## Builds Debian metapackage for Qubes Workstation base dependencies
@@ -22,11 +23,11 @@ securedrop-workstation-viewer: ## Builds Debian metapackage for Disposable VM de
 
 .PHONY: securedrop-export
 securedrop-export: ## Builds Debian package for Qubes Workstation export scripts
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-export" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-export" ./scripts/build-debianpackage
 
 .PHONY: securedrop-log
 securedrop-log: ## Builds Debian package for Qubes Workstation securedrop-log scripts
-	WHEELS_DIR="$(PWD)/localwheels/" PKG_NAME="securedrop-log" ./scripts/build-debianpackage
+	PKG_NAME="securedrop-log" ./scripts/build-debianpackage
 
 .PHONY: securedrop-keyring
 securedrop-keyring: ## Builds Debian package containing the release key
@@ -46,9 +47,9 @@ requirements: ## Creates requirements files for the Python projects
 
 .PHONY: build-wheels
 build-wheels: ## Builds the wheels and adds them to the localwheels directory
-	./scripts/verify-sha256sum-signature
-	./scripts/build-sync-wheels -p ${PKG_DIR}
-	./scripts/sync-sha256sums
+	./scripts/verify-sha256sum-signature $$(basename ${PKG_DIR})
+	./scripts/build-sync-wheels
+	./scripts/sync-sha256sums $$(basename ${PKG_DIR})
 	@printf "Done! Now please follow the instructions in\n"
 	@printf "https://github.com/freedomofpress/securedrop-debian-packaging-guide/"
 	@printf "to push these changes to the FPF PyPI index\n"
@@ -57,10 +58,6 @@ build-wheels: ## Builds the wheels and adds them to the localwheels directory
 test: ## Run simple test suite (skips reproducibility checks)
 	pytest -v tests/test_update_requirements.py
 
-.PHONY: clean
-clean: ## Removes all non-version controlled packaging artifacts
-	rm -rf localwheels/*
-
 .PHONY: reprotest
 reprotest: ## Runs only reproducibility tests, for .deb and .whl files
 	pytest -vvs tests/test_reproducible_*.py

README.md

@@ -40,20 +40,21 @@ If we have to update the tool, use the following steps
 rm -rf .venv && python3 -m venv .venv
 source .venv/bin/activate
 # Then install pip-tools, from pinned dependencies
-python3 -m pip install -r requirements.txt
+python3 -m pip install -r workstation-bootstrap/requirements.txt
 # Then update the requirements.in file as required
-pip-compile --allow-unsafe --generate-hashes --output-file=requirements.txt requirements.in
+pip-compile --allow-unsafe --generate-hashes \
+    --output-file=workstation-bootstrap/requirements.txt workstation-bootstrap/requirements.in
 # Now we are ready for bootstrapping
-./scripts/build-sync-wheels --cache ./bootstrap -p $PWD
+./scripts/build-sync-wheels --project workstation-bootstrap --pkg-dir ./workstation-bootstrap --requirements .
 # Here we have the new wheels ready
 # Now let us recreate our new sha256sums for bootstrapping
-BOOTSTRAP=true ./scripts/sync-sha256sums
+./scripts/sync-sha256sums ./workstation-bootstrap
 # now let us sign the list of sha256sums
-gpg --armor --output bootstrap-sha256sums.txt.asc --detach-sig  bootstrap-sha256sums.txt
+gpg --armor --output workstation-bootstrap/sha256sums.txt.asc --detach-sig  workstation-bootstrap/sha256sums.txt
 # We can even verify if we want
-BOOTSTRAP=true ./scripts/verify-sha256sum-signature
+./scripts/verify-sha256sum-signature ./workstation-bootstrap/
 # Update the build-requirements.txt file
-PKG_DIR=$PWD BOOTSTRAP=true ./scripts/update-requirements
+./scripts/update-requirements --pkg-dir ./workstation-bootstrap/ --project workstation-bootstrap --requirements .
 ```
 
 Make sure that your GPG public key is stored in `pubkeys/`, so CI can verify the signatures.

scripts/build-debianpackage

@@ -12,15 +12,26 @@ set -e
 set -u
 set -o pipefail
 
+
+# Validate required args.
+if [[ -z "${PKG_NAME:-}" ]]; then
+    echo "Set PKG_NAME of the build";
+    exit 1
+fi
+
 # Store root of repo, since we'll change dirs several times.
 CUR_DIR="$(git rev-parse --show-toplevel)"
 VERSION_CODENAME=$("${CUR_DIR}/scripts/codename")
 
-# Verify sha256sums.txt in the git repo
-"${CUR_DIR}/scripts/verify-sha256sum-signature"
-
 # Disable use of pip cache during debhelper build actions.
 export DH_PIP_EXTRA_ARGS="--no-cache-dir --require-hashes"
+# Point dh-virtualenv/pip to our prebuilt wheels
+export WHEELS_DIR="${CUR_DIR}/${PKG_NAME}/wheels"
+
+if [[ -d "${WHEELS_DIR}" ]]; then
+    # Verify sha256sums.txt in the git repo if we have dependencies
+    "${CUR_DIR}/scripts/verify-sha256sum-signature" "${PKG_NAME}"
+fi
 
 # Declare general packaging building workspace; subdirs will
 # be created within, to build specific packages.
@@ -29,12 +40,6 @@ mkdir -p "$TOP_BUILDDIR"
 rm -rf "${TOP_BUILDDIR:?}/${PKG_NAME}"
 mkdir -p "${TOP_BUILDDIR}/${PKG_NAME}"
 
-# Validate required args.
-if [[ -z "${PKG_NAME:-}" ]]; then
-    echo "Set PKG_NAME of the build";
-    exit 1
-fi
-
 
 # Look up most recent release from GitHub repo
 function find_latest_version() {
@@ -115,8 +120,10 @@ if [[ "${PKG_NAME}" =~ ^(securedrop-client|securedrop-proxy|securedrop-export|se
 
     # Hop into the package build dir, to run dpkg-buildpackage
     cd "$TOP_BUILDDIR/$PKG_NAME/"
-    # Verify all the hashes from the verified sha256sums.txt
-    "${CUR_DIR}/scripts/verify-hashes" "${CUR_DIR}/sha256sums.txt"
+    if [[ -d "${WHEELS_DIR}" ]]; then
+        # Verify all the hashes from the verified sha256sums.txt if we have dependencies
+        "${CUR_DIR}/scripts/verify-hashes" "${CUR_DIR}/${PKG_NAME}/sha256sums.txt"
+    fi
 
     echo "All hashes verified."
 else
@@ -158,7 +165,7 @@ export SOURCE_DATE_EPOCH
 # Build the package
 dpkg-buildpackage -us -uc
 
-# Tell the user the path of the files buillt
+# Tell the user the path of the files built
 pkg_path="$(find "$TOP_BUILDDIR" -type f -iname "${PKG_NAME}_${PKG_VERSION}*.deb" | head -n1)"
 if [[ -f "$pkg_path" ]]; then
     echo "Package location: $pkg_path"

scripts/build-sync-wheels

@@ -5,10 +5,8 @@ import sys
 import glob
 import subprocess
 import tempfile
-import tarfile
 import shutil
 import argparse
-from pprint import pprint
 
 
 if os.geteuid() == 0:
@@ -35,38 +33,40 @@ WHEEL_BUILD_DIR = "/tmp/pip-wheel-build"
 
 
 def main():
+    if "PKG_DIR" in os.environ and \
+            not any(arg.startswith("--pkg-dir") for arg in sys.argv):
+        sys.argv.extend(["--pkg-dir", os.environ["PKG_DIR"]])
+        sys.argv.extend(["--project", os.path.basename(os.environ["PKG_DIR"])])
+
     parser = argparse.ArgumentParser(
-        description="Builds and caches sources and wheels"
-    )
-    parser.add_argument(
-        "-p",
-        help="Points to the project directory",
-    )
-    parser.add_argument(
-        "--cache", default="./localwheels", help="Final cache dir"
+        description="Builds and stores sources and wheels"
     )
+    parser.add_argument("--pkg-dir", help="Package directory", required=True)
+    parser.add_argument("--project", help="Project name to update", required=True)
     parser.add_argument(
         "--clobber", action="store_true", default=False,
         help="Whether to overwrite wheels and source tarballs",
     )
+    parser.add_argument(
+        "--requirements",
+        default="requirements",
+        help="Directory that contains requirements.txt inside the package directory",
+    )
     args = parser.parse_args()
 
-    if args.p.startswith("https://"):
+    if args.pkg_dir.startswith("https://"):
         git_clone_directory = tempfile.mkdtemp(prefix=os.path.basename(args.p))
-        cmd = f"git clone {args.p} {git_clone_directory}".split()
+        cmd = f"git clone {args.pkg_dir} {git_clone_directory}".split()
         subprocess.check_call(cmd)
-        args.p = git_clone_directory
+        args.pkg_dir = git_clone_directory
     else:
         git_clone_directory = ""
-        if not os.path.exists(args.p):
-            print("Project directory missing {0}.".format(args.p))
+        if not os.path.exists(args.pkg_dir):
+            print(f"Project directory missing {args.pkg_dir}.")
             sys.exit(1)
 
-    # Try requirements.txt in the repo root, otherwise try requirements/requirements.txt
-    req_path = os.path.join(args.p, "requirements.txt")
-
-    if not os.path.exists(req_path):
-        req_path = os.path.join(args.p, "requirements/requirements.txt")
+    req_path = os.path.join(args.pkg_dir, args.requirements, "requirements.txt")
+    local_wheels = os.path.join(args.project, "wheels")
 
     if not os.path.exists(req_path):
         print("requirements.txt missing at {0}.".format(req_path))
@@ -98,9 +98,6 @@ def main():
         # Now we have all the source tarballs
         source_tar_balls = glob.glob(f"{tmpdir}/*.tar.gz")
         for source in source_tar_balls:
-            # I am getting umask issue for the below lines
-            # tar = tarfile.open(source)
-            # tar.extractall(WHEEL_BUILD_DIR)
             cmd = ["tar", "-xvf", source, "-C", WHEEL_BUILD_DIR]
             subprocess.check_call(cmd)
         # Now we have the all source tarballs extracted in WHEEL_BUILD_DIR
@@ -114,23 +111,26 @@ def main():
             subprocess.check_call(cmd)
             print(f"build command used: {' '.join(cmd)}")
 
-
         # Now find the names
         names = os.listdir(tmpdir)
 
-        cachenames = os.listdir(args.cache)
+        if os.path.exists(local_wheels):
+            local_names = os.listdir(local_wheels)
+        else:
+            os.mkdir(local_wheels)
+            local_names = []
 
         for name in names:
-            if name == "requirements.txt":  # We don't need this in cache
+            if name == "requirements.txt":  # We don't need this
                 continue
-            if name in cachenames:  # Means all ready in our cache
+            if name in local_names:  # Means already there
                 if not args.clobber:
                     continue
 
-            # Else copy to cache
+            # Else copy to local wheels
             filepath = os.path.join(tmpdir, name)
-            shutil.copy(filepath, args.cache, follow_symlinks=True)
-            print("Copying {0} to cache {1}".format(name, args.cache))
+            shutil.copy(filepath, local_wheels, follow_symlinks=True)
+            print(f"Copied {name} to {local_wheels}")
 
     if git_clone_directory:
         shutil.rmtree(git_clone_directory)

scripts/install-deps

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euxo pipefail
-# Installs required dependencies for building SecureDrop Worsktation packages.
-# Assumes a Debian 10 machine, ideally a Qubes AppVM.
+# Installs required dependencies for building SecureDrop Workstation packages.
+# Assumes a Debian 11 machine, ideally a Qubes AppVM.
 
 VIRTUAL_ENV="${VIRTUAL_ENV:-}"
 
@@ -26,7 +26,7 @@ sudo apt-get install  \
 
 # Inspect the wheel files present locally. If repo was cloned
 # without git-lfs, they'll be "text/plain", rather than "application/zip".
-wheel_mime_types="$(find localwheels/ -type f -iname '*.whl' -exec file --mime-type {} + | perl -F':\s+' -lanE 'say $F[-1]' | sort -u)"
+wheel_mime_types="$(find workstation-bootstrap/ -type f -iname '*.whl' -exec file --mime-type {} + | perl -F':\s+' -lanE 'say $F[-1]' | sort -u)"
 if [[ "$wheel_mime_types" != "application/zip" ]]; then
     echo "Re-fetching git-lfs assets..."
     git lfs install
@@ -42,5 +42,7 @@ else
     echo "Virtualenv already activated, skipping creation..."
 fi
 
-# Install the 'build' tool from previously prepared localwheels
-pip install --require-hashes --no-index --no-deps --no-cache-dir -r build-requirements.txt --find-links ./bootstrap/
+# Install the 'build' tool from previously prepared bootstrap
+pip install --require-hashes --no-index --no-deps --no-cache-dir \
+    -r ./workstation-bootstrap/build-requirements.txt \
+    --find-links ./workstation-bootstrap/wheels/

scripts/sync-sha256sums

@@ -2,24 +2,12 @@
 
 # A script to update the sha256sums from localwheels directory
 
-set -e
-set -u
-set -o pipefail
-set -o nounset
+set -euxo pipefail
 
+directory=${1?"Usage: $0 [directory]"}
 
-bootstrap=${BOOTSTRAP:-}
+cd ./${directory}/wheels/
+sha256sum * > ../sha256sums.txt
 
-if [ ! -z "$bootstrap" ]; then
-	cd ./bootstrap/
-	sha256sum * > ../bootstrap-sha256sums.txt
-
-	echo "Now you must sign the generated bootstrap-sha256sums.txt file:"
-	echo "gpg --armor --output bootstrap-sha256sums.txt.asc --detach-sig  bootstrap-sha256sums.txt"
-else
-	cd ./localwheels/
-	sha256sum * > ../sha256sums.txt
-
-	echo "Now you must sign the generated sha256sums.txt file:"
-	echo "gpg --armor --output sha256sums.txt.asc --detach-sig  sha256sums.txt"
-fi
+echo "Now you must sign the generated sha256sums.txt file:"
+echo "gpg --armor --output ${directory}/sha256sums.txt.asc --detach-sig  ${directory}/sha256sums.txt"