From cbb0755f71e4eab8853abdf27d093ab2a6eadcdb Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Thu, 30 Sep 2021 14:52:16 +0100 Subject: [PATCH 01/16] Add `--run-background-updates` option to `update_database` script. Signed-off-by: Nick Barrett --- scripts-dev/update_database | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/scripts-dev/update_database b/scripts-dev/update_database index 87f709b6ed43..9f69d6660f34 100755 --- a/scripts-dev/update_database +++ b/scripts-dev/update_database @@ -56,6 +56,12 @@ if __name__ == "__main__": required=True, help="A database config file for either a SQLite3 database or a PostgreSQL one.", ) + parser.add_argument( + "--run-background-updates", + type=bool, + required=False, + help="Whether to run background updates after upgrading the database schema.", + ) args = parser.parse_args() @@ -82,19 +88,21 @@ if __name__ == "__main__": # Setup instantiates the store within the homeserver object and updates the # DB. hs.setup() - store = hs.get_datastore() - async def run_background_updates(): - await store.db_pool.updates.run_background_updates(sleep=False) - # Stop the reactor to exit the script once every background update is run. - reactor.stop() + if args.run_background_updates: + store = hs.get_datastore() - def run(): - # Apply all background updates on the database. - defer.ensureDeferred( - run_as_background_process("background_updates", run_background_updates) - ) + async def run_background_updates(): + await store.db_pool.updates.run_background_updates(sleep=False) + # Stop the reactor to exit the script once every background update is run. + reactor.stop() + + def run(): + # Apply all background updates on the database. + defer.ensureDeferred( + run_as_background_process("background_updates", run_background_updates) + ) - reactor.callWhenRunning(run) + reactor.callWhenRunning(run) - reactor.run() + reactor.run() From ebd19320245c0f020fe72ff0096ee7543e28a23b Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Thu, 30 Sep 2021 15:05:05 +0100 Subject: [PATCH 02/16] Add changelog for PR #10954. --- changelog.d/10954.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/10954.feature diff --git a/changelog.d/10954.feature b/changelog.d/10954.feature new file mode 100644 index 000000000000..122610bdaa83 --- /dev/null +++ b/changelog.d/10954.feature @@ -0,0 +1 @@ +Add `--run-background-updates` option to `update_database` script. From d3daf7181cb24a556cd16973b984414660ae0b23 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 3 Oct 2021 16:56:57 +0100 Subject: [PATCH 03/16] Move running background updates into separate function. --- scripts-dev/update_database | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/scripts-dev/update_database b/scripts-dev/update_database index 9f69d6660f34..9f3650bd4f8b 100755 --- a/scripts-dev/update_database +++ b/scripts-dev/update_database @@ -42,6 +42,25 @@ class MockHomeserver(HomeServer): self.version_string = "Synapse/" + get_version_string(synapse) +def run_background_updates(hs): + store = hs.get_datastore() + + async def run_background_updates(): + await store.db_pool.updates.run_background_updates(sleep=False) + # Stop the reactor to exit the script once every background update is run. + reactor.stop() + + def run(): + # Apply all background updates on the database. + defer.ensureDeferred( + run_as_background_process("background_updates", run_background_updates) + ) + + reactor.callWhenRunning(run) + + reactor.run() + + if __name__ == "__main__": parser = argparse.ArgumentParser( description=( @@ -90,19 +109,4 @@ if __name__ == "__main__": hs.setup() if args.run_background_updates: - store = hs.get_datastore() - - async def run_background_updates(): - await store.db_pool.updates.run_background_updates(sleep=False) - # Stop the reactor to exit the script once every background update is run. - reactor.stop() - - def run(): - # Apply all background updates on the database. - defer.ensureDeferred( - run_as_background_process("background_updates", run_background_updates) - ) - - reactor.callWhenRunning(run) - - reactor.run() + run_background_updates(hs) From 6da417509229531e7112530b8bb21b6cd1656ddf Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 3 Oct 2021 16:57:54 +0100 Subject: [PATCH 04/16] Move main script block into separate function. --- scripts-dev/update_database | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts-dev/update_database b/scripts-dev/update_database index 9f3650bd4f8b..567cb7fd0ebe 100755 --- a/scripts-dev/update_database +++ b/scripts-dev/update_database @@ -61,7 +61,7 @@ def run_background_updates(hs): reactor.run() -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser( description=( "Updates a synapse database to the latest schema and runs background updates" @@ -110,3 +110,7 @@ if __name__ == "__main__": if args.run_background_updates: run_background_updates(hs) + + +if __name__ == "__main__": + main() From 57d6d44245e59e6bea421f2809caa3d69c761687 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 3 Oct 2021 16:59:03 +0100 Subject: [PATCH 05/16] Add attribution to changelog entry. --- changelog.d/10954.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/10954.feature b/changelog.d/10954.feature index 122610bdaa83..fb69874a6e09 100644 --- a/changelog.d/10954.feature +++ b/changelog.d/10954.feature @@ -1 +1 @@ -Add `--run-background-updates` option to `update_database` script. +Add `--run-background-updates` option to `update_database` script. Contributed by @Fizzadar at Beeper. From a9462fe9ec99bc00d6991918e5e7f3ebed4e1af5 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 3 Oct 2021 17:00:10 +0100 Subject: [PATCH 06/16] Move DB update script to `scripts/update_synapse_database`. --- .ci/scripts/test_synapse_port_db.sh | 4 ++-- scripts-dev/lint.sh | 2 +- scripts-dev/make_full_schema.sh | 2 +- .../update_database => scripts/update_synapse_database | 0 tox.ini | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename scripts-dev/update_database => scripts/update_synapse_database (100%) diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh index 2b4e5ec1707d..5e79675f6ca1 100755 --- a/.ci/scripts/test_synapse_port_db.sh +++ b/.ci/scripts/test_synapse_port_db.sh @@ -25,7 +25,7 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml echo "--- Prepare test database" # Make sure the SQLite3 database is using the latest schema and has no pending background update. -scripts-dev/update_database --database-config .ci/sqlite-config.yaml +scripts/update_synapse_database --database-config .ci/sqlite-config.yaml # Create the PostgreSQL database. .ci/scripts/postgres_exec.py "CREATE DATABASE synapse" @@ -46,7 +46,7 @@ echo "--- Prepare empty SQLite database" # we do this by deleting the sqlite db, and then doing the same again. rm .ci/test_db.db -scripts-dev/update_database --database-config .ci/sqlite-config.yaml +scripts/update_synapse_database --database-config .ci/sqlite-config.yaml # re-create the PostgreSQL database. .ci/scripts/postgres_exec.py \ diff --git a/scripts-dev/lint.sh b/scripts-dev/lint.sh index 809eff166ab2..b6554a73c115 100755 --- a/scripts-dev/lint.sh +++ b/scripts-dev/lint.sh @@ -90,10 +90,10 @@ else "scripts/hash_password" "scripts/register_new_matrix_user" "scripts/synapse_port_db" + "scripts/update_synapse_database" "scripts-dev" "scripts-dev/build_debian_packages" "scripts-dev/sign_json" - "scripts-dev/update_database" "contrib" "synctl" "setup.py" "synmark" "stubs" ".ci" ) fi diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 39bf30d258bd..5e5ae51e6095 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -147,7 +147,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG" # Make sure the SQLite3 database is using the latest schema and has no pending background update. echo "Running db background jobs..." -scripts-dev/update_database --database-config "$SQLITE_CONFIG" +scripts/update_synapse_database --database-config "$SQLITE_CONFIG" # Create the PostgreSQL database. echo "Creating postgres database..." diff --git a/scripts-dev/update_database b/scripts/update_synapse_database similarity index 100% rename from scripts-dev/update_database rename to scripts/update_synapse_database diff --git a/tox.ini b/tox.ini index 5a62ec76c23f..cfe6a0694269 100644 --- a/tox.ini +++ b/tox.ini @@ -41,10 +41,10 @@ lint_targets = scripts/hash_password scripts/register_new_matrix_user scripts/synapse_port_db + scripts/update_synapse_database scripts-dev scripts-dev/build_debian_packages scripts-dev/sign_json - scripts-dev/update_database stubs contrib synctl From 7cabb6c857dc7dbbcd4d9267812c1f2527672443 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 3 Oct 2021 17:03:46 +0100 Subject: [PATCH 07/16] Ensure background updates are run in synapse port db test. --- .ci/scripts/test_synapse_port_db.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/scripts/test_synapse_port_db.sh b/.ci/scripts/test_synapse_port_db.sh index 5e79675f6ca1..50115b3079a8 100755 --- a/.ci/scripts/test_synapse_port_db.sh +++ b/.ci/scripts/test_synapse_port_db.sh @@ -25,7 +25,7 @@ python -m synapse.app.homeserver --generate-keys -c .ci/sqlite-config.yaml echo "--- Prepare test database" # Make sure the SQLite3 database is using the latest schema and has no pending background update. -scripts/update_synapse_database --database-config .ci/sqlite-config.yaml +scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates # Create the PostgreSQL database. .ci/scripts/postgres_exec.py "CREATE DATABASE synapse" @@ -46,7 +46,7 @@ echo "--- Prepare empty SQLite database" # we do this by deleting the sqlite db, and then doing the same again. rm .ci/test_db.db -scripts/update_synapse_database --database-config .ci/sqlite-config.yaml +scripts/update_synapse_database --database-config .ci/sqlite-config.yaml --run-background-updates # re-create the PostgreSQL database. .ci/scripts/postgres_exec.py \ From 12e4d11de76ea155b93c56137b4350d3c627869e Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 3 Oct 2021 17:04:40 +0100 Subject: [PATCH 08/16] Add `updates/update_synapse_database` to Debian packaging config. --- debian/matrix-synapse-py3.links | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/matrix-synapse-py3.links b/debian/matrix-synapse-py3.links index 53e29654187a..7eeba180d903 100644 --- a/debian/matrix-synapse-py3.links +++ b/debian/matrix-synapse-py3.links @@ -3,3 +3,4 @@ opt/venvs/matrix-synapse/bin/register_new_matrix_user usr/bin/register_new_matri opt/venvs/matrix-synapse/bin/synapse_port_db usr/bin/synapse_port_db opt/venvs/matrix-synapse/bin/synapse_review_recent_signups usr/bin/synapse_review_recent_signups opt/venvs/matrix-synapse/bin/synctl usr/bin/synctl +opt/venvs/matrix-synapse/bin/update_synapse_database usr/bin/update_synapse_database From 8e0a888c10e314a5345a25d1a83083006e30f671 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 4 Oct 2021 14:30:53 +0100 Subject: [PATCH 09/16] Update help text for Synapse configuration file. Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- scripts/update_synapse_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database index 567cb7fd0ebe..50f259b3a648 100755 --- a/scripts/update_synapse_database +++ b/scripts/update_synapse_database @@ -73,7 +73,7 @@ def main(): "--database-config", type=argparse.FileType("r"), required=True, - help="A database config file for either a SQLite3 database or a PostgreSQL one.", + help="Synapse configuration file, giving the details of the database to be updated", ) parser.add_argument( "--run-background-updates", From c56d2a4f3e93e02aa2661b7a29b2b85f365717be Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 4 Oct 2021 14:31:50 +0100 Subject: [PATCH 10/16] Update help text for background updates flag. Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- scripts/update_synapse_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database index 50f259b3a648..77a4841a8ad6 100755 --- a/scripts/update_synapse_database +++ b/scripts/update_synapse_database @@ -79,7 +79,7 @@ def main(): "--run-background-updates", type=bool, required=False, - help="Whether to run background updates after upgrading the database schema.", + help="run background updates after upgrading the database schema", ) args = parser.parse_args() From 2d5780caad2bb3d1f3d450310b5d24dbba0f4c27 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 4 Oct 2021 14:32:27 +0100 Subject: [PATCH 11/16] Update changelog. Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/10954.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/10954.feature b/changelog.d/10954.feature index fb69874a6e09..94dfa7175c31 100644 --- a/changelog.d/10954.feature +++ b/changelog.d/10954.feature @@ -1 +1 @@ -Add `--run-background-updates` option to `update_database` script. Contributed by @Fizzadar at Beeper. +Include an `update_synapse_database` script in the distribution. Contributed by @Fizzadar at Beeper. From 2bf28821c1ef3025f6cb078e033ddad28ef6a170 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 4 Oct 2021 14:32:49 +0100 Subject: [PATCH 12/16] Update text for update synapase database script. Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- scripts/update_synapse_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database index 77a4841a8ad6..dbff78b99bd0 100755 --- a/scripts/update_synapse_database +++ b/scripts/update_synapse_database @@ -64,7 +64,7 @@ def run_background_updates(hs): def main(): parser = argparse.ArgumentParser( description=( - "Updates a synapse database to the latest schema and runs background updates" + "Updates a synapse database to the latest schema and optionally runs background updates" " on it." ) ) From 8c3d0e5b9b4cb129d3a01b2cb709fc3987d4190a Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 4 Oct 2021 14:33:34 +0100 Subject: [PATCH 13/16] Add `--run-background-updates` flag in `make_full_schema.sh` script. --- scripts-dev/make_full_schema.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts-dev/make_full_schema.sh b/scripts-dev/make_full_schema.sh index 5e5ae51e6095..c3c90f4ec637 100755 --- a/scripts-dev/make_full_schema.sh +++ b/scripts-dev/make_full_schema.sh @@ -147,7 +147,7 @@ python -m synapse.app.homeserver --generate-keys -c "$SQLITE_CONFIG" # Make sure the SQLite3 database is using the latest schema and has no pending background update. echo "Running db background jobs..." -scripts/update_synapse_database --database-config "$SQLITE_CONFIG" +scripts/update_synapse_database --database-config --run-background-updates "$SQLITE_CONFIG" # Create the PostgreSQL database. echo "Creating postgres database..." From b270bf99d26e2018ada3b2d0a8a3817aeefc8eb4 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Mon, 4 Oct 2021 14:33:41 +0100 Subject: [PATCH 14/16] Add Debian package changelog entry. --- debian/changelog | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debian/changelog b/debian/changelog index b08a5927808d..f6282aab8677 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,10 @@ +matrix-synapse-py3 (1.44.0~rc2+nmu1) UNRELEASED; urgency=medium + + [ Nick @ Beeper ] + * Include an `update_synapse_database` script in the distribution. + + -- root Mon, 04 Oct 2021 13:29:26 +0000 + matrix-synapse-py3 (1.44.0~rc2) stable; urgency=medium * New synapse release 1.44.0~rc2. From fb90b632b816a8c89131caf6c5ae5508b99ec923 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 5 Oct 2021 14:46:22 +0100 Subject: [PATCH 15/16] Fix argparse flag in `update_synapse_database` script. --- scripts/update_synapse_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database index dbff78b99bd0..be6d61f1d6cc 100755 --- a/scripts/update_synapse_database +++ b/scripts/update_synapse_database @@ -77,7 +77,7 @@ def main(): ) parser.add_argument( "--run-background-updates", - type=bool, + action='store_true', required=False, help="run background updates after upgrading the database schema", ) From 54597899b802efca023d97bcbd8df381e3393352 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Tue, 5 Oct 2021 20:10:57 +0100 Subject: [PATCH 16/16] Fix quotes. --- scripts/update_synapse_database | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/update_synapse_database b/scripts/update_synapse_database index be6d61f1d6cc..26b29b0b4593 100755 --- a/scripts/update_synapse_database +++ b/scripts/update_synapse_database @@ -77,7 +77,7 @@ def main(): ) parser.add_argument( "--run-background-updates", - action='store_true', + action="store_true", required=False, help="run background updates after upgrading the database schema", )