Merge remote-tracking branch 'origin/develop' into erikj/auth_chains_…

…alog

.buildkite/scripts/create_postgres_db.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import logging
+
 from synapse.storage.engines import create_engine
 
 logger = logging.getLogger("create_postgres_db")

INSTALL.md

@@ -257,7 +257,7 @@ for a number of platforms.
 
 #### Docker images and Ansible playbooks
 
-There is an offical synapse image available at
+There is an official synapse image available at
 <https://hub.docker.com/r/matrixdotorg/synapse> which can be used with
 the docker-compose file available at [contrib/docker](contrib/docker). Further
 information on this including configuration options is available in the README

README.rst

@@ -243,7 +243,7 @@ Then update the ``users`` table in the database::
 Synapse Development
 ===================
 
-Join our developer community on Matrix: [#synapse-dev:matrix.org](https://matrix.to/#/#synapse-dev:matrix.org)
+Join our developer community on Matrix: `#synapse-dev:matrix.org <https://matrix.to/#/#synapse-dev:matrix.org>`_
 
 Before setting up a development environment for synapse, make sure you have the
 system dependencies (such as the python header files) installed - see

changelog.d/9016.misc

@@ -0,0 +1 @@
+Ensure rejected events get added to some metadata tables.

changelog.d/9024.feature

@@ -0,0 +1 @@
+Improved performance when calculating ignored users in large rooms.

changelog.d/9025.misc

@@ -0,0 +1 @@
+Removed an unused column from `access_tokens` table.

changelog.d/9028.bugfix

@@ -0,0 +1 @@
+Fix a long-standing bug where some caches could grow larger than configured.

changelog.d/9030.misc

@@ -0,0 +1 @@
+Add a `-noextras` factor to `tox.ini`, to support running the tests with no optional dependencies.

changelog.d/9031.misc

@@ -0,0 +1 @@
+Fix running unit tests when optional dependencies are not installed.

changelog.d/9033.misc

@@ -0,0 +1 @@
+Allow bumping schema version when using split out state database.

changelog.d/9035.doc

@@ -0,0 +1 @@
+Corrected a typo in the `systemd-with-workers` documentation.

changelog.d/9036.feature

@@ -0,0 +1 @@
+Add support for multiple SSO Identity Providers.

changelog.d/9038.misc

@@ -0,0 +1 @@
+Configure the linters to run on a consistent set of files.

changelog.d/9039.removal

@@ -0,0 +1 @@
+Remove broken and unmaintained `demo/webserver.py` script.

changelog.d/9040.doc

@@ -0,0 +1 @@
+Corrected a typo in `INSTALL.md`.

changelog.d/9041.misc

@@ -0,0 +1 @@
+Various cleanups to device inbox store.

changelog.d/9042.feature

@@ -0,0 +1 @@
+Add experimental support for handling and persistence of to-device messages to happen on worker processes.

changelog.d/9043.feature

@@ -0,0 +1 @@
+Add experimental support for handling and persistence of to-device messages to happen on worker processes.

changelog.d/9044.feature

@@ -0,0 +1 @@
+Add experimental support for handling and persistence of to-device messages to happen on worker processes.

changelog.d/9051.bugfix

@@ -0,0 +1 @@
+Fix error handling during insertion of client IPs into the database.

changelog.d/9053.bugfix

@@ -0,0 +1 @@
+Fix bug where we didn't correctly record CPU time spent in 'on_new_event' block.

changelog.d/9054.bugfix

@@ -0,0 +1 @@
+Fix a minor bug which could cause confusing error messages from invalid configurations.

changelog.d/9055.misc

@@ -0,0 +1 @@
+Drop unused database tables.

changelog.d/9057.doc

@@ -0,0 +1 @@
+Add missing user_mapping_provider configuration to the Keycloak OIDC example. Contributed by @chris-ruecker.

changelog.d/9058.misc

@@ -0,0 +1 @@
+Remove unused `SynapseService` class.

docs/openid.md

@@ -158,6 +158,10 @@ oidc_config:
    client_id: "synapse"
    client_secret: "copy secret generated from above"
    scopes: ["openid", "profile"]
+   user_mapping_provider:
+     config:
+       localpart_template: "{{ user.preferred_username }}"
+       display_name_template: "{{ user.name }}"
 ```
 ### [Auth0][auth0]
 

docs/systemd-with-workers/README.md

@@ -31,7 +31,7 @@ There is no need for a separate configuration file for the master process.
 1. Adjust synapse configuration files as above.
 1. Copy the `*.service` and `*.target` files in [system](system) to
 `/etc/systemd/system`.
-1. Run `systemctl deamon-reload` to tell systemd to load the new unit files.
+1. Run `systemctl daemon-reload` to tell systemd to load the new unit files.
 1. Run `systemctl enable matrix-synapse.service`. This will configure the
 synapse master process to be started as part of the `matrix-synapse.target`
 target.

mypy.ini

@@ -103,6 +103,7 @@ files =
   tests/replication,
   tests/test_utils,
   tests/handlers/test_password_providers.py,
+  tests/rest/client/v1/test_login.py,
   tests/rest/client/v2_alpha/test_auth.py,
   tests/util/test_stream_change_cache.py
 

scripts/synapse_port_db

@@ -629,6 +629,7 @@ class Porter(object):
             await self._setup_state_group_id_seq()
             await self._setup_user_id_seq()
             await self._setup_events_stream_seqs()
+            await self._setup_device_inbox_seq()
 
             # Step 3. Get tables.
             self.progress.set_state("Fetching tables")
@@ -911,6 +912,32 @@ class Porter(object):
             "_setup_events_stream_seqs", _setup_events_stream_seqs_set_pos,
         )
 
+    async def _setup_device_inbox_seq(self):
+        """Set the device inbox sequence to the correct value.
+        """
+        curr_local_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
+            table="device_inbox",
+            keyvalues={},
+            retcol="COALESCE(MAX(stream_id), 1)",
+            allow_none=True,
+        )
+
+        curr_federation_id = await self.sqlite_store.db_pool.simple_select_one_onecol(
+            table="device_federation_outbox",
+            keyvalues={},
+            retcol="COALESCE(MAX(stream_id), 1)",
+            allow_none=True,
+        )
+
+        next_id = max(curr_local_id, curr_federation_id) + 1
+
+        def r(txn):
+            txn.execute(
+                "ALTER SEQUENCE device_inbox_sequence RESTART WITH %s", (next_id,)
+            )
+
+        return self.postgres_store.db_pool.runInteraction("_setup_device_inbox_seq", r)
+
 
 ##############################################
 # The following is simply UI stuff

stubs/frozendict.pyi

@@ -15,16 +15,7 @@
 
 # Stub for frozendict.
 
-from typing import (
-    Any,
-    Hashable,
-    Iterable,
-    Iterator,
-    Mapping,
-    overload,
-    Tuple,
-    TypeVar,
-)
+from typing import Any, Hashable, Iterable, Iterator, Mapping, Tuple, TypeVar, overload
 
 _KT = TypeVar("_KT", bound=Hashable)  # Key type.
 _VT = TypeVar("_VT")  # Value type.

stubs/sortedcontainers/sorteddict.pyi

@@ -7,17 +7,17 @@ from typing import (
     Callable,
     Dict,
     Hashable,
-    Iterator,
-    Iterable,
     ItemsView,
+    Iterable,
+    Iterator,
     KeysView,
     List,
     Mapping,
     Optional,
     Sequence,
+    Tuple,
     Type,
     TypeVar,
-    Tuple,
     Union,
     ValuesView,
     overload,

stubs/txredisapi.pyi

@@ -16,7 +16,7 @@
 """Contains *incomplete* type hints for txredisapi.
 """
 
-from typing import List, Optional, Union, Type
+from typing import List, Optional, Type, Union
 
 class RedisProtocol:
     def publish(self, channel: str, message: bytes): ...

synapse/app/generic_worker.py

@@ -108,6 +108,7 @@
 )
 from synapse.rest.client.v2_alpha.keys import KeyChangesServlet, KeyQueryServlet
 from synapse.rest.client.v2_alpha.register import RegisterRestServlet
+from synapse.rest.client.v2_alpha.sendtodevice import SendToDeviceRestServlet
 from synapse.rest.client.versions import VersionsRestServlet
 from synapse.rest.health import HealthResource
 from synapse.rest.key.v2 import KeyApiV2Resource
@@ -520,6 +521,8 @@ def _listen_http(self, listener_config: ListenerConfig):
                     room.register_deprecated_servlets(self, resource)
                     InitialSyncRestServlet(self).register(resource)
 
+                    SendToDeviceRestServlet(self).register(resource)
+
                     user_directory.register_servlets(self, resource)
 
                     # If presence is disabled, use the stub servlet that does

synapse/app/homeserver.py

@@ -15,13 +15,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import gc
 import logging
 import os
 import sys
 from typing import Iterable, Iterator
 
-from twisted.application import service
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
 from twisted.web.resource import EncodingResourceWrapper, IResource
@@ -73,7 +71,6 @@
 from synapse.util.httpresourcetree import create_resource_tree
 from synapse.util.manhole import manhole
 from synapse.util.module_loader import load_module
-from synapse.util.rlimit import change_resource_limit
 from synapse.util.versionstring import get_version_string
 
 logger = logging.getLogger("synapse.app.homeserver")
@@ -487,25 +484,6 @@ def format_config_error(e: ConfigError) -> Iterator[str]:
         e = e.__cause__
 
 
-class SynapseService(service.Service):
-    """
-    A twisted Service class that will start synapse. Used to run synapse
-    via twistd and a .tac.
-    """
-
-    def __init__(self, config):
-        self.config = config
-
-    def startService(self):
-        hs = setup(self.config)
-        change_resource_limit(hs.config.soft_file_limit)
-        if hs.config.gc_thresholds:
-            gc.set_threshold(*hs.config.gc_thresholds)
-
-    def stopService(self):
-        return self._port.stopListening()
-
-
 def run(hs):
     PROFILE_SYNAPSE = False
     if PROFILE_SYNAPSE:

synapse/config/_util.py

@@ -56,7 +56,7 @@ def json_error_to_config_error(
     """
     # copy `config_path` before modifying it.
     path = list(config_path)
-    for p in list(e.path):
+    for p in list(e.absolute_path):
         if isinstance(p, int):
             path.append("<item %i>" % p)
         else:

synapse/config/workers.py

@@ -53,6 +53,9 @@ class WriterLocations:
         default=["master"], type=List[str], converter=_instance_to_list_converter
     )
     typing = attr.ib(default="master", type=str)
+    to_device = attr.ib(
+        default=["master"], type=List[str], converter=_instance_to_list_converter,
+    )
 
 
 class WorkerConfig(Config):
@@ -124,7 +127,7 @@ def read_config(self, config, **kwargs):
 
         # Check that the configured writers for events and typing also appears in
         # `instance_map`.
-        for stream in ("events", "typing"):
+        for stream in ("events", "typing", "to_device"):
             instances = _instance_to_list_converter(getattr(self.writers, stream))
             for instance in instances:
                 if instance != "master" and instance not in self.instance_map:
@@ -133,6 +136,11 @@ def read_config(self, config, **kwargs):
                         % (instance, stream)
                     )
 
+        if len(self.writers.to_device) != 1:
+            raise ConfigError(
+                "Must only specify one instance to handle `to_device` messages."
+            )
+
         self.events_shard_config = ShardedWorkerHandlingConfig(self.writers.events)
 
         # Whether this worker should run background tasks or not.