Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Use a chain cover index to efficiently calculate auth chain difference #8868

Merged
merged 48 commits into from
Jan 11, 2021
Merged
Changes from 1 commit
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
49e888d
Change alg
erikjohnston Dec 1, 2020
8c760ff
Calculate chain ID/seq no on event insertion
erikjohnston Dec 2, 2020
85348e1
Add some docs about the chain cover
erikjohnston Dec 2, 2020
02d1198
Handle old rooms
erikjohnston Dec 3, 2020
61ab47e
Fix schema for sqlite
erikjohnston Dec 3, 2020
6141825
Fix up _get_auth_chain_difference_using_chains_txn
erikjohnston Dec 3, 2020
c7e2ce5
Newsfile
erikjohnston Dec 3, 2020
66e779d
Add type
erikjohnston Dec 3, 2020
cf2243f
Fixup
erikjohnston Dec 3, 2020
bd30c9e
Fix take1
erikjohnston Dec 4, 2020
55f03b9
Fixup
erikjohnston Dec 4, 2020
3e98fb7
More fixups
erikjohnston Dec 4, 2020
9087033
Newsfile
erikjohnston Dec 4, 2020
21b3ef0
Test both new and old methods
erikjohnston Dec 4, 2020
fdaf4da
Note
erikjohnston Dec 4, 2020
7f5ac13
isort
erikjohnston Dec 4, 2020
afb7f80
Don't add links where start and end chain are the same
erikjohnston Dec 7, 2020
dec1f74
Have exists_path_from handle same chain case correctly
erikjohnston Dec 7, 2020
9279940
Add some tests
erikjohnston Dec 7, 2020
6a74e21
Fix unit tests on postgres
erikjohnston Dec 7, 2020
654eff1
Add missing 'auth'
erikjohnston Dec 7, 2020
dbecefd
Fixup typing for execute_values
erikjohnston Dec 8, 2020
123b431
Rename _get_auth_chain_difference_using_chains_txn and add comment
erikjohnston Dec 8, 2020
883e922
Add some definitions
erikjohnston Dec 8, 2020
988f25a
Fixup link confusion
erikjohnston Dec 8, 2020
08ec78b
Make para less dense (hopefully)
erikjohnston Dec 8, 2020
024c802
Add note about auth chain
erikjohnston Dec 8, 2020
4cc769f
Be explicit
erikjohnston Dec 8, 2020
7d75efb
rm variant
erikjohnston Dec 8, 2020
92b5e4b
Add note about current algo
erikjohnston Dec 8, 2020
a9552c2
Update docs/auth_chain_difference_algorithm.md
erikjohnston Dec 8, 2020
7cc6d7e
Fix up _LinkMap
erikjohnston Dec 8, 2020
5fa05f2
Fix up event_chain tests
erikjohnston Dec 8, 2020
8dac80c
Merge remote-tracking branch 'origin/develop' into erikj/auth_chains_…
erikjohnston Dec 9, 2020
e3d0be4
Make sorted_topologically stable and add tests
erikjohnston Dec 9, 2020
cdb88c2
Make _LinkMap use tuples
erikjohnston Dec 9, 2020
0f91c86
Review comments
erikjohnston Dec 9, 2020
888450a
Fix typo
erikjohnston Dec 9, 2020
c9422b6
Handle rooms the server used to be in correctly.
erikjohnston Jan 5, 2021
c8758af
Handle case where we don't have chain info for an event
erikjohnston Jan 6, 2021
b2ac553
Merge remote-tracking branch 'origin/develop' into erikj/auth_chains_…
erikjohnston Jan 6, 2021
d96264d
Merge remote-tracking branch 'origin/develop' into erikj/auth_chains_…
erikjohnston Jan 6, 2021
d64f5f8
Typo
erikjohnston Jan 11, 2021
6071eff
Split out a has_auth_chain_index
erikjohnston Jan 11, 2021
368d3b8
Update docstring
erikjohnston Jan 11, 2021
bea2c47
Move to schema 59
erikjohnston Jan 11, 2021
03dd636
Merge remote-tracking branch 'origin/develop' into erikj/auth_chains_…
erikjohnston Jan 11, 2021
8c1e32c
Fix tests after merge from develop
erikjohnston Jan 6, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 33 additions & 16 deletions synapse/storage/databases/main/room.py
Original file line number Diff line number Diff line change
Expand Up @@ -1166,6 +1166,37 @@ def set_room_is_public(self, room_id, is_public):
# It's overridden by RoomStore for the synapse master.
raise NotImplementedError()

async def has_auth_chain_index(self, room_id: str) -> bool:
"""Check if the room has (or can have) a chain cover index.

Defaults to True if we don't have an entry in `rooms` table nor any
events for the room.
"""

has_auth_chain_index = await self.db_pool.simple_select_one_onecol(
table="rooms",
keyvalues={"room_id": room_id},
retcol="has_auth_chain_index",
desc="has_auth_chain_index",
allow_none=True,
)

if has_auth_chain_index:
return True

# It's possible that we already have events for the room in our DB
# without a corresponding room entry. If we do then we don't want to
# mark the room as having an auth chain cover index.
max_ordering = await self.db_pool.simple_select_one_onecol(
table="events",
keyvalues={"room_id": room_id},
retcol="MAX(stream_ordering)",
allow_none=True,
desc="upsert_room_on_join",
)

return max_ordering is None


class RoomStore(RoomBackgroundUpdateStore, RoomWorkerStore, SearchStore):
def __init__(self, database: DatabasePool, db_conn, hs):
Expand All @@ -1182,14 +1213,7 @@ async def upsert_room_on_join(self, room_id: str, room_version: RoomVersion):
# It's possible that we already have events for the room in our DB
# without a corresponding room entry. If we do then we don't want to
# mark the room as having an auth chain cover index.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We probably don't need this comment anymore (and the one near line 1267) as it is in the has_auth_chain_index method now.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think its a bit unclear why we're bothering to call the function otherwise?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't feel strongly. I'm OK with leaving it.

max_ordering = await self.db_pool.simple_select_one_onecol(
table="events",
keyvalues={"room_id": room_id},
retcol="MAX(stream_ordering)",
allow_none=True,
desc="upsert_room_on_join",
)
has_auth_chain_index = max_ordering is None
has_auth_chain_index = await self.has_auth_chain_index(room_id)

await self.db_pool.simple_upsert(
desc="upsert_room_on_join",
Expand Down Expand Up @@ -1267,14 +1291,7 @@ async def maybe_store_room_on_outlier_membership(
# It's possible that we already have events for the room in our DB
# without a corresponding room entry. If we do then we don't want to
# mark the room as having an auth chain cover index.
max_ordering = await self.db_pool.simple_select_one_onecol(
table="events",
keyvalues={"room_id": room_id},
retcol="MAX(stream_ordering)",
allow_none=True,
desc="maybe_store_room_on_outlier_membership",
)
has_auth_chain_index = max_ordering is None
has_auth_chain_index = await self.has_auth_chain_index(room_id)

await self.db_pool.simple_upsert(
desc="maybe_store_room_on_outlier_membership",
Expand Down