From 7f9841bdec349075eb424c943db5569439acb83c Mon Sep 17 00:00:00 2001 From: Brendan Abolivier Date: Wed, 24 Nov 2021 20:21:44 +0100 Subject: [PATCH] Lower minumum batch size to 1 for background updates (#11422) Co-authored-by: Richard van der Hoff <1389908+richvdh@users.noreply.github.com> --- changelog.d/11422.bugfix | 1 + synapse/storage/background_updates.py | 2 +- tests/rest/admin/test_background_updates.py | 25 ++++++++++++++------- tests/storage/test_background_update.py | 8 +++---- 4 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 changelog.d/11422.bugfix diff --git a/changelog.d/11422.bugfix b/changelog.d/11422.bugfix new file mode 100644 index 000000000000..28ac65ea7ce3 --- /dev/null +++ b/changelog.d/11422.bugfix @@ -0,0 +1 @@ +Improve performance of various background database schema updates. diff --git a/synapse/storage/background_updates.py b/synapse/storage/background_updates.py index b104f9032cf0..bc8364400d2d 100644 --- a/synapse/storage/background_updates.py +++ b/synapse/storage/background_updates.py @@ -82,7 +82,7 @@ class BackgroundUpdater: process and autotuning the batch size. """ - MINIMUM_BACKGROUND_BATCH_SIZE = 100 + MINIMUM_BACKGROUND_BATCH_SIZE = 1 DEFAULT_BACKGROUND_BATCH_SIZE = 100 BACKGROUND_UPDATE_INTERVAL_MS = 1000 BACKGROUND_UPDATE_DURATION_MS = 100 diff --git a/tests/rest/admin/test_background_updates.py b/tests/rest/admin/test_background_updates.py index 1786316763c0..cd5c60b65c1f 100644 --- a/tests/rest/admin/test_background_updates.py +++ b/tests/rest/admin/test_background_updates.py @@ -20,6 +20,7 @@ from synapse.api.errors import Codes from synapse.rest.client import login from synapse.server import HomeServer +from synapse.storage.background_updates import BackgroundUpdater from tests import unittest @@ -150,9 +151,11 @@ def test_status_bg_update(self): "current_updates": { "master": { "name": "test_update", - "average_items_per_ms": 0.1, + "average_items_per_ms": 0.001, "total_duration_ms": 1000.0, - "total_item_count": 100, + "total_item_count": ( + BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + ), } }, "enabled": True, @@ -203,9 +206,11 @@ def test_enabled(self): "current_updates": { "master": { "name": "test_update", - "average_items_per_ms": 0.1, + "average_items_per_ms": 0.001, "total_duration_ms": 1000.0, - "total_item_count": 100, + "total_item_count": ( + BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + ), } }, "enabled": False, @@ -230,9 +235,11 @@ def test_enabled(self): "current_updates": { "master": { "name": "test_update", - "average_items_per_ms": 0.1, + "average_items_per_ms": 0.001, "total_duration_ms": 1000.0, - "total_item_count": 100, + "total_item_count": ( + BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + ), } }, "enabled": False, @@ -267,9 +274,11 @@ def test_enabled(self): "current_updates": { "master": { "name": "test_update", - "average_items_per_ms": 0.1, + "average_items_per_ms": 0.001, "total_duration_ms": 2000.0, - "total_item_count": 200, + "total_item_count": ( + 2 * BackgroundUpdater.MINIMUM_BACKGROUND_BATCH_SIZE + ), } }, "enabled": True, diff --git a/tests/storage/test_background_update.py b/tests/storage/test_background_update.py index 0da42b5ac559..a5f5ebad410f 100644 --- a/tests/storage/test_background_update.py +++ b/tests/storage/test_background_update.py @@ -19,11 +19,11 @@ def prepare(self, reactor, clock, homeserver): ) def test_do_background_update(self): - # the time we claim each update takes - duration_ms = 42 + # the time we claim it takes to update one item when running the update + duration_ms = 4200 # the target runtime for each bg update - target_background_update_duration_ms = 50000 + target_background_update_duration_ms = 5000000 store = self.hs.get_datastore() self.get_success( @@ -57,7 +57,7 @@ async def update(progress, count): # on the first call, we should get run with the default background update size self.update_handler.assert_called_once_with( - {"my_key": 1}, self.updates.DEFAULT_BACKGROUND_BATCH_SIZE + {"my_key": 1}, self.updates.MINIMUM_BACKGROUND_BATCH_SIZE ) # second step: complete the update