From ca17814c0cdb0625606bbf53c811b7a494937bba Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 14:35:29 +0100 Subject: [PATCH 1/6] Use `poetry` to build venv in debian packages Co-authored-by: Dan Callahan Co-authored-by: Shay --- debian/build_virtualenv | 21 ++++++++++++++++++--- debian/changelog | 7 +++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index e6911636192c..46e1492fa6b2 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -30,8 +30,22 @@ case $(dpkg-architecture -q DEB_HOST_ARCH) in ;; esac -# Use --builtin-venv to use the better `venv` module from CPython 3.4+ rather -# than the 2/3 compatible `virtualenv`. +# Manually install Poetry and export a pip-compatible `requirements.txt` +# We need a Poetry pre-release as the export command is buggy in < 1.2 +if [ -e requirements.txt ]; then + # Guard against a possible future where requirements.txt lives in the repo. + # Otherwise, calling `poetry export` below would silently clobber it. + echo "requirements.txt already exists; aborting" 1>&2 + exit 1 +fi +TEMP_VENV="$(mktemp -d)" +python3 -m venv "$TEMP_VENV" +source "$TEMP_VENV/bin/activate" +pip install -U pip +pip install poetry==1.2.0b1 +poetry export --extras "all test" -o requirements.txt +deactivate +rm -rf "$TEMP_VENV" dh_virtualenv \ --install-suffix "matrix-synapse" \ @@ -43,7 +57,8 @@ dh_virtualenv \ --preinstall="wheel" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ - --extras="all,systemd,test" + --extras="all,systemd,test" \ + --requirements="requirements.txt" PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" diff --git a/debian/changelog b/debian/changelog index 903d98af02b4..80ad43173786 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.56.0+nmu1) UNRELEASED; urgency=medium + + * Non-maintainer upload. + * Use poetry to manage the bundled virtualenv included with this package. + + -- Synapse Packaging Team Wed, 30 Mar 2022 12:21:43 +0100 + matrix-synapse-py3 (1.56.0) stable; urgency=medium * New synapse release 1.56.0. From 111e28f3460fbfaf5a67d296f62be260425784b1 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 14:42:58 +0100 Subject: [PATCH 2/6] Changelog --- changelog.d/12449.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/12449.misc diff --git a/changelog.d/12449.misc b/changelog.d/12449.misc new file mode 100644 index 000000000000..03e08aace427 --- /dev/null +++ b/changelog.d/12449.misc @@ -0,0 +1 @@ +Use `poetry` to manage the virtualenv in debian packages. From fd7b3b3e6dc550ec16e64d53ab1d3df049e33559 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 14:52:27 +0100 Subject: [PATCH 3/6] Only pull in from requirements.txt Addresses the same problem as #12439. --- debian/build_virtualenv | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 46e1492fa6b2..377fbf79a044 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -55,6 +55,7 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ + --extra-pip-arg "--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ --extras="all,systemd,test" \ From 1a4c5352b46326ec5ba8f32fdfae4da68df25680 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 16:33:19 +0100 Subject: [PATCH 4/6] Include `test` and `all` extras `poetry export` helpfully silently ignores an unknown extra Haven't seen this before because it's the only place we export `all` and `test`. I could have __sworm__ that the syntax `--extra "all test"` worked for `poetry install`... --- debian/build_virtualenv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 377fbf79a044..1d94dce7f502 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -43,7 +43,7 @@ python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip pip install poetry==1.2.0b1 -poetry export --extras "all test" -o requirements.txt +poetry export --extras all --extras test -o requirements.txt deactivate rm -rf "$TEMP_VENV" From a6e7e3b451e48a180306b91e9fb287706e9fdb2a Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 17:12:00 +0100 Subject: [PATCH 5/6] Clean up requirements file on subsequence builds --- debian/build_virtualenv | 12 ++++-------- debian/clean | 1 + 2 files changed, 5 insertions(+), 8 deletions(-) create mode 100644 debian/clean diff --git a/debian/build_virtualenv b/debian/build_virtualenv index 1d94dce7f502..f012d9076fc0 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -32,18 +32,12 @@ esac # Manually install Poetry and export a pip-compatible `requirements.txt` # We need a Poetry pre-release as the export command is buggy in < 1.2 -if [ -e requirements.txt ]; then - # Guard against a possible future where requirements.txt lives in the repo. - # Otherwise, calling `poetry export` below would silently clobber it. - echo "requirements.txt already exists; aborting" 1>&2 - exit 1 -fi TEMP_VENV="$(mktemp -d)" python3 -m venv "$TEMP_VENV" source "$TEMP_VENV/bin/activate" pip install -U pip pip install poetry==1.2.0b1 -poetry export --extras all --extras test -o requirements.txt +poetry export --extras all --extras test -o exported_requirements.txt deactivate rm -rf "$TEMP_VENV" @@ -55,11 +49,13 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ + # Use --no-deps to only install pinned versions in exported_requirements.txt, + # and to avoid https://github.com/pypa/pip/issues/9644 --extra-pip-arg "--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ --extras="all,systemd,test" \ - --requirements="requirements.txt" + --requirements="exported_requirements.txt" PACKAGE_BUILD_DIR="debian/matrix-synapse-py3" VIRTUALENV_DIR="${PACKAGE_BUILD_DIR}${DH_VIRTUALENV_INSTALL_ROOT}/matrix-synapse" diff --git a/debian/clean b/debian/clean new file mode 100644 index 000000000000..d488f298d587 --- /dev/null +++ b/debian/clean @@ -0,0 +1 @@ +exported_requirements.txt From dad410484c2f746a6ff056f630c7085222f84914 Mon Sep 17 00:00:00 2001 From: David Robertson Date: Tue, 12 Apr 2022 19:48:14 +0100 Subject: [PATCH 6/6] Fix shell syntax --- debian/build_virtualenv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debian/build_virtualenv b/debian/build_virtualenv index f012d9076fc0..b068792592d6 100755 --- a/debian/build_virtualenv +++ b/debian/build_virtualenv @@ -41,6 +41,8 @@ poetry export --extras all --extras test -o exported_requirements.txt deactivate rm -rf "$TEMP_VENV" +# Use --no-deps to only install pinned versions in exported_requirements.txt, +# and to avoid https://github.com/pypa/pip/issues/9644 dh_virtualenv \ --install-suffix "matrix-synapse" \ --builtin-venv \ @@ -49,9 +51,7 @@ dh_virtualenv \ --preinstall="lxml" \ --preinstall="mock" \ --preinstall="wheel" \ - # Use --no-deps to only install pinned versions in exported_requirements.txt, - # and to avoid https://github.com/pypa/pip/issues/9644 - --extra-pip-arg "--no-deps" \ + --extra-pip-arg="--no-deps" \ --extra-pip-arg="--no-cache-dir" \ --extra-pip-arg="--compile" \ --extras="all,systemd,test" \