Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Track device list updates per room. #12321

Merged
merged 30 commits into from Apr 4, 2022
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
b5a9c6b
Operate on room IDs
erikjohnston Mar 25, 2022
6fa639e
Don't copy the hosts list
erikjohnston Mar 28, 2022
ecf98b9
Write to a new `device_lists_changes_in_room` table.
erikjohnston Mar 28, 2022
c5dd83f
Convert to doing everything in a transaction
erikjohnston Mar 28, 2022
bb44214
Track if we've calculated remote hosts
erikjohnston Mar 28, 2022
98fceb3
We actually don't want stream_id to be unique
erikjohnston Mar 28, 2022
eda0e64
Handle room pokes that haven't been converted to outbound pokes
erikjohnston Mar 29, 2022
c7790ab
Deduplicate outbound pokes
erikjohnston Mar 29, 2022
6e9b31a
Add a config option that allows using new code path
erikjohnston Mar 29, 2022
3b2ab93
Add tests for new code path
erikjohnston Mar 29, 2022
f8af30f
Newsfile
erikjohnston Mar 29, 2022
7266580
Fix tests
erikjohnston Mar 29, 2022
f24b70b
Merge remote-tracking branch 'origin/develop' into erikj/device_list_…
erikjohnston Mar 31, 2022
8bd8ee2
Update synapse/storage/schema/main/delta/69/01device_list_oubound_by_…
erikjohnston Apr 4, 2022
56f0913
Update synapse/storage/schema/main/delta/69/01device_list_oubound_by_…
erikjohnston Apr 4, 2022
90d41a0
Encode opentracing context just once.
erikjohnston Apr 4, 2022
c470a12
Rename var
erikjohnston Apr 4, 2022
d030062
Remove `if not room_ids` check.
erikjohnston Apr 4, 2022
ad5d46b
Add unique index
erikjohnston Apr 4, 2022
d5031b0
Note lack of foreign key constraint
erikjohnston Apr 4, 2022
28dacc8
Add comment about stream_id duplicates
erikjohnston Apr 4, 2022
f48527f
Update synapse_port_db
erikjohnston Apr 4, 2022
bd45f19
Inequality the wrong way round
erikjohnston Apr 4, 2022
dee8f55
Add note about 'num_stream_ids'
erikjohnston Apr 4, 2022
3574541
Merge remote-tracking branch 'origin/develop' into erikj/device_list_…
erikjohnston Apr 4, 2022
cf04f1a
Use different stream IDs for device_list_outbound_pokes
erikjohnston Apr 4, 2022
89e10d7
Correctly order device list stream updates
erikjohnston Apr 4, 2022
e54d2d4
Wake up replication after adding otubound pokes
erikjohnston Apr 4, 2022
7d79dee
Apply suggestions from code review
erikjohnston Apr 4, 2022
b61c5c7
Remove get_users_who_share_room_with_user stub in test
erikjohnston Apr 4, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
13 changes: 5 additions & 8 deletions synapse/storage/databases/main/devices.py
Expand Up @@ -1531,7 +1531,6 @@ async def add_device_change_to_streams(

num_stream_ids = max(
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it might be good to add a comment clarifying why this is a max rather than a total (vis, that we use the same set of stream ids for each of the three tables)

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The max got replaced, but I've added a comment the equivalent code

len(device_ids),
len(room_ids) * len(device_ids),
len(hosts) * len(device_ids),
)

Expand Down Expand Up @@ -1670,29 +1669,27 @@ def _add_device_outbound_room_poke_txn(
stream_ids: List[str],
context: Dict[str, str],
) -> None:
next_stream_id = iter(stream_ids)

self.db_pool.simple_insert_many_txn(
txn,
table="device_lists_changes_in_room",
keys=(
"stream_id",
"room_id",
"user_id",
"device_id",
"room_id",
"stream_id",
"converted_to_destinations",
"opentracing_context",
),
values=[
(
next(next_stream_id),
room_id,
user_id,
device_id,
room_id,
stream_id,
True, # As we're updating `device_lists_outbound_pokes` at the same time.
json_encoder.encode(context),
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
)
for room_id in room_ids
for device_id in device_ids
for device_id, stream_id in zip(device_ids, stream_ids)
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
],
)
Expand Up @@ -14,15 +14,15 @@
*/

CREATE TABLE device_lists_changes_in_room (
stream_id BIGINT NOT NULL,
room_id TEXT NOT NULL,
user_id TEXT NOT NULL,
device_id TEXT NOT NULL,
room_id TEXT NOT NULL,
stream_id BIGINT NOT NULL, -- This matches `device_lists_stream.stream_id`
richvdh marked this conversation as resolved.
Show resolved Hide resolved
-- We track if we've calculated the hosts in the room and updated
-- `device_lists_outbound_pokes`.
converted_to_destinations BOOLEAN NOT NULL,
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
opentracing_context TEXT
);

CREATE UNIQUE INDEX device_lists_changes_in_stream_id ON device_lists_changes_in_room(stream_id);
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved
CREATE INDEX device_lists_changes_in_stream_id ON device_lists_changes_in_room(stream_id);
CREATE INDEX device_lists_changes_in_stream_id_converted ON device_lists_changes_in_room(stream_id) WHERE NOT converted_to_destinations;
erikjohnston marked this conversation as resolved.
Show resolved Hide resolved