Add endpoints for backfilling history (MSC2716)

changelog.d/9247.feature

@@ -0,0 +1 @@
+Add experimental support for backfilling history into rooms ([MSC2716](https://github.com/matrix-org/matrix-doc/pull/2716)).

scripts-dev/complement.sh

@@ -11,12 +11,10 @@ cd "$(dirname $0)/.."
 docker build -t matrixdotorg/synapse:latest -f docker/Dockerfile .
 
 # Download Complement
-wget -N https://github.com/matrix-org/complement/archive/master.tar.gz
-tar -xzf master.tar.gz
-cd complement-master
+cd ../complement
 
 # Build the Synapse image from Complement, based on the above image we just built
 docker build -t complement-synapse -f dockerfiles/Synapse.Dockerfile ./dockerfiles
 
 # Run the tests on the resulting image!
-COMPLEMENT_BASE_IMAGE=complement-synapse go test -v -count=1 ./tests
+COMPLEMENT_BASE_IMAGE=complement-synapse go test -tags msc2716 -v -count=1 ./tests/main_test.go ./tests/msc2716_test.go

synapse/config/experimental.py

@@ -27,3 +27,6 @@ def read_config(self, config: JsonDict, **kwargs):
 
         # MSC2858 (multiple SSO identity providers)
         self.msc2858_enabled = experimental.get("msc2858_enabled", False)  # type: bool
+
+        # MSC2716 (backfill existing history)
+        self.msc2716_enabled = experimental.get("msc2716_enabled", False)  # type: bool

synapse/events/builder.py

@@ -101,6 +101,7 @@ async def build(
         self,
         prev_event_ids: List[str],
         auth_event_ids: Optional[List[str]],
+        inherit_depth: bool,
     ) -> EventBase:
         """Transform into a fully signed and hashed event
 
@@ -109,6 +110,8 @@ async def build(
             auth_event_ids: The event IDs to use as the auth events.
                 Should normally be set to None, which will cause them to be calculated
                 based on the room state at the prev_events.
+            inherit_depth: True to use the oldest depth from the prev_event_ids
+                (instead of calculating a new depth).
 
         Returns:
             The signed and hashed event.
@@ -133,7 +136,12 @@ async def build(
             prev_events = prev_event_ids
 
         old_depth = await self._store.get_max_depth_of(prev_event_ids)
-        depth = old_depth + 1
+        # If backfilling old message, use the same depth as what we're inserting next to.
+        if inherit_depth:
+            depth = old_depth
+        # Otherwise, progress the depth as normal
+        else:
+            depth = old_depth + 1
 
         # we cap depth of generated events, to ensure that they are not
         # rejected by other servers (and so that they can be persisted in

synapse/handlers/message.py

@@ -446,6 +446,7 @@ async def create_event(
         event_dict: dict,
         txn_id: Optional[str] = None,
         prev_event_ids: Optional[List[str]] = None,
+        inherit_depth: bool = False,
         auth_event_ids: Optional[List[str]] = None,
         require_consent: bool = True,
     ) -> Tuple[EventBase, EventContext]:
@@ -467,6 +468,9 @@ async def create_event(
 
                 If None, they will be requested from the database.
 
+            inherit_depth: True to use the oldest depth from the prev_event_ids
+                (instead of calculating a new depth).
+
             auth_event_ids:
                 The event ids to use as the auth_events for the new event.
                 Should normally be left as None, which will cause them to be calculated
@@ -533,6 +537,7 @@ async def create_event(
             builder=builder,
             requester=requester,
             prev_event_ids=prev_event_ids,
+            inherit_depth=inherit_depth,
             auth_event_ids=auth_event_ids,
         )
 
@@ -690,6 +695,7 @@ async def create_and_send_nonmember_event(
         self,
         requester: Requester,
         event_dict: dict,
+        inherit_depth: bool = False,
         ratelimit: bool = True,
         txn_id: Optional[str] = None,
         ignore_shadow_ban: bool = False,
@@ -702,6 +708,8 @@ async def create_and_send_nonmember_event(
         Args:
             requester: The requester sending the event.
             event_dict: An entire event.
+            inherit_depth: True to use the oldest depth from the event_dict["prev_events"]
+                (instead of calculating a new depth).
             ratelimit: Whether to rate limit this send.
             txn_id: The transaction ID.
             ignore_shadow_ban: True if shadow-banned users should be allowed to
@@ -744,8 +752,14 @@ async def create_and_send_nonmember_event(
                     assert event.internal_metadata.stream_ordering
                     return event, event.internal_metadata.stream_ordering
 
+            prev_events = event_dict.get("prev_events")
+
             event, context = await self.create_event(
-                requester, event_dict, txn_id=txn_id
+                requester,
+                event_dict,
+                txn_id=txn_id,
+                prev_event_ids=prev_events,
+                inherit_depth=inherit_depth,
             )
 
             assert self.hs.is_mine_id(event.sender), "User must be our own: %s" % (
@@ -776,6 +790,7 @@ async def create_new_client_event(
         builder: EventBuilder,
         requester: Optional[Requester] = None,
         prev_event_ids: Optional[List[str]] = None,
+        inherit_depth: bool = False,
         auth_event_ids: Optional[List[str]] = None,
     ) -> Tuple[EventBase, EventContext]:
         """Create a new event for a local client
@@ -789,6 +804,9 @@ async def create_new_client_event(
 
                 If None, they will be requested from the database.
 
+            inherit_depth: True to use the oldest depth from the prev_event_ids
+                (instead of calculating a new depth).
+
             auth_event_ids:
                 The event ids to use as the auth_events for the new event.
                 Should normally be left as None, which will cause them to be calculated
@@ -817,7 +835,9 @@ async def create_new_client_event(
         ), "Attempting to create an event with no prev_events"
 
         event = await builder.build(
-            prev_event_ids=prev_event_ids, auth_event_ids=auth_event_ids
+            prev_event_ids=prev_event_ids,
+            auth_event_ids=auth_event_ids,
+            inherit_depth=inherit_depth,
         )
         context = await self.state.compute_event_context(event)
         if requester:
@@ -1403,6 +1423,7 @@ async def _rebuild_event_after_third_party_rules(
         event = await builder.build(
             prev_event_ids=original_event.prev_event_ids(),
             auth_event_ids=original_event.auth_event_ids(),
+            inherit_depth=False,
         )
 
         # we rebuild the event context, to be on the safe side. If nothing else,

synapse/handlers/sync.py

@@ -42,7 +42,7 @@
 from synapse.util.caches.lrucache import LruCache
 from synapse.util.caches.response_cache import ResponseCache
 from synapse.util.metrics import Measure, measure_func
-from synapse.visibility import filter_events_for_client
+from synapse.visibility import filter_events_for_client, filter_historical_events
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -259,6 +259,8 @@ def __init__(self, hs: "HomeServer"):
             expiry_ms=LAZY_LOADED_MEMBERS_CACHE_MAX_AGE,
         )
 
+        self._msc2716_enabled = hs.config.experimental.msc2716_enabled
+
     async def wait_for_sync_for_user(
         self,
         requester: Requester,
@@ -458,6 +460,10 @@ async def _load_filtered_recents(
                     recents,
                     always_include_ids=current_state_ids,
                 )
+
+                if self._msc2716_enabled:
+                    # `m.historical` events should not come down /sync
+                    recents = filter_historical_events(recents)
             else:
                 recents = []
 
@@ -521,6 +527,10 @@ async def _load_filtered_recents(
                 loaded_recents.extend(recents)
                 recents = loaded_recents
 
+                if self._msc2716_enabled:
+                    # `m.historical` events should not come down /sync
+                    recents = filter_historical_events(recents)
+
                 if len(events) <= load_limit:
                     limited = False
                     break