Merge commit '837293c31' into anoa/dinsic_release_1_21_x

* commit '837293c31':
  Remove obsolete __future__ imports (#8337)
  Use admin_patterns for all admin APIs. (#8331)
  Fix a potential bug of UnboundLocalError (#8329)
  Switch metaclass initialization to python 3-compatible syntax (#8326)
  Catch-up after Federation Outage (split, 4): catch-up loop (#8272)
  Use slots in attrs classes where possible (#8296)
  Fix typos in comments.
  Add the topic and avatar to the room details admin API (#8305)
  Improve SAML error messages (#8248)
  Add experimental support for sharding event persister. Again. (#8294)
  Make `StreamToken.room_key` be a `RoomStreamToken` instance. (#8281)
  Use TLSv1.2 for fake servers in tests (#8208)
  Add /_synapse/client to the reverse proxy docs (#8227)
  Clean up `Notifier.on_new_room_event` code path (#8288)

UPGRADE.rst

@@ -128,38 +128,6 @@ template. These templates are similar, but the parameters are slightly different
 * A string ``error`` parameter is available that includes a short hint of why a
   user is seeing the error page.
 
-ThirdPartyEventRules breaking changes
--------------------------------------
-
-This release introduces a backwards-incompatible change to modules making use of
-`ThirdPartyEventRules` in Synapse.
-
-The `http_client` argument is no longer passed to modules as they are initialised. Instead,
-modules are expected to make use of the `http_client` property on the `ModuleApi` class.
-Modules are now passed a `module_api` argument during initialisation, which is an instance of
-`ModuleApi`.
-
-New HTML templates
-------------------
-
-A new HTML template,
-`password_reset_confirmation.html <https://github.com/matrix-org/synapse/blob/develop/synapse/res/templates/password_reset_confirmation.html>`_,
-has been added to the ``synapse/res/templates`` directory. If you are using a
-custom template directory, you may want to copy the template over and modify it.
-
-Note that as of v1.20.0, templates do not need to be included in custom template
-directories for Synapse to start. The default templates will be used if a custom
-template cannot be found.
-
-This page will appear to the user after clicking a password reset link that has
-been emailed to them.
-
-To complete password reset, the page must include a way to make a `POST`
-request to
-``/_synapse/client/password_reset/{medium}/submit_token``
-with the query parameters from the original link, presented as a URL-encoded form. See the file
-itself for more details.
-
 Upgrading to v1.18.0
 ====================
 

changelog.d/8208.misc

@@ -0,0 +1 @@
+Fix tests on distros which disable TLSv1.0. Contributed by @danc86.

changelog.d/8227.doc

@@ -0,0 +1 @@
+Add `/_synapse/client` to the reverse proxy documentation.

changelog.d/8248.feature

@@ -0,0 +1 @@
+Consolidate the SSO error template across all configuration.

changelog.d/8272.bugfix

@@ -0,0 +1 @@
+Fix messages over federation being lost until an event is sent into the same room.

changelog.d/8281.misc

@@ -0,0 +1 @@
+Change `StreamToken.room_key` to be a `RoomStreamToken` instance.

changelog.d/8288.misc

@@ -0,0 +1 @@
+Refactor notifier code to correctly use the max event stream position.

changelog.d/8294.feature

@@ -0,0 +1 @@
+Add experimental support for sharding event persister.

changelog.d/8296.misc

@@ -0,0 +1 @@
+Use slotted classes where possible.

changelog.d/8305.feature

@@ -0,0 +1 @@
+Add the room topic and avatar to the room details admin API.

changelog.d/8324.bugfix

@@ -0,0 +1 @@
+Fix fetching events from remote servers that are malformed.

changelog.d/8326.misc

@@ -0,0 +1 @@
+Update outdated usages of `metaclass` to python 3 syntax.

changelog.d/8329.bugfix

@@ -0,0 +1 @@
+Fix UnboundLocalError from occuring when appservices send malformed register request.

changelog.d/8331.misc

@@ -0,0 +1 @@
+Use the `admin_patterns` helper in additional locations.

changelog.d/8337.misc

@@ -0,0 +1 @@
+Remove `__future__` imports related to Python 2 compatibility.

contrib/cmdclient/console.py

@@ -15,8 +15,6 @@
 # limitations under the License.
 
 """ Starts a synapse client console. """
-from __future__ import print_function
-
 import argparse
 import cmd
 import getpass

contrib/cmdclient/http.py

@@ -13,8 +13,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import json
 import urllib
 from pprint import pformat

contrib/graph/graph.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 import cgi
 import datetime

contrib/graph/graph3.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import argparse
 import cgi
 import datetime

contrib/jitsimeetbridge/jitsimeetbridge.py

@@ -10,8 +10,6 @@
 Requires:
 npm install jquery jsdom
 """
-from __future__ import print_function
-
 import json
 import subprocess
 import time

contrib/scripts/kick_users.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-from __future__ import print_function
 
 import json
 import sys
@@ -8,11 +7,6 @@
 
 import requests
 
-try:
-    raw_input
-except NameError:  # Python 3
-    raw_input = input
-
 
 def _mkurl(template, kws):
     for key in kws:
@@ -58,7 +52,7 @@ def main(hs, room_id, access_token, user_id_prefix, why):
     print("The following user IDs will be kicked from %s" % room_name)
     for uid in kick_list:
         print(uid)
-    doit = raw_input("Continue? [Y]es\n")
+    doit = input("Continue? [Y]es\n")
     if len(doit) > 0 and doit.lower() == "y":
         print("Kicking members...")
         # encode them all

docs/admin_api/rooms.md

@@ -275,6 +275,8 @@ The following fields are possible in the JSON response body:
 
 * `room_id` - The ID of the room.
 * `name` - The name of the room.
+* `topic` - The topic of the room.
+* `avatar` - The `mxc` URI to the avatar of the room.
 * `canonical_alias` - The canonical (main) alias address of the room.
 * `joined_members` - How many users are currently in the room.
 * `joined_local_members` - How many local users are currently in the room.
@@ -304,6 +306,8 @@ Response:
 {
   "room_id": "!mscvqgqpHYjBGDxNym:matrix.org",
   "name": "Music Theory",
+  "avatar": "mxc://matrix.org/AQDaVFlbkQoErdOgqWRgiGSV",
+  "topic": "Theory, Composition, Notation, Analysis",
   "canonical_alias": "#musictheory:matrix.org",
   "joined_members": 127
   "joined_local_members": 2,

docs/reverse_proxy.md

@@ -11,7 +11,7 @@ privileges.
 
 **NOTE**: Your reverse proxy must not `canonicalise` or `normalise`
 the requested URI in any way (for example, by decoding `%xx` escapes).
-Beware that Apache *will* canonicalise URIs unless you specifify
+Beware that Apache *will* canonicalise URIs unless you specify
 `nocanon`.
 
 When setting up a reverse proxy, remember that Matrix clients and other
@@ -23,6 +23,10 @@ specification](https://matrix.org/docs/spec/server_server/latest#resolving-serve
 for more details of the algorithm used for federation connections, and
 [delegate.md](<delegate.md>) for instructions on setting up delegation.
 
+Endpoints that are part of the standardised Matrix specification are
+located under `/_matrix`, whereas endpoints specific to Synapse are
+located under `/_synapse/client`.
+
 Let's assume that we expect clients to connect to our server at
 `https://matrix.example.com`, and other servers to connect at
 `https://example.com:8448`.  The following sections detail the configuration of
@@ -45,7 +49,7 @@ server {
 
     server_name matrix.example.com;
 
-    location /_matrix {
+    location ~* ^(\/_matrix|\/_synapse\/client) {
         proxy_pass http://localhost:8008;
         proxy_set_header X-Forwarded-For $remote_addr;
         # Nginx by default only allows file uploads up to 1M in size
@@ -65,6 +69,10 @@ matrix.example.com {
   proxy /_matrix http://localhost:8008 {
     transparent
   }
+
+  proxy /_synapse/client http://localhost:8008 {
+    transparent
+  }
 }
 
 example.com:8448 {
@@ -79,6 +87,7 @@ example.com:8448 {
 ```
 matrix.example.com {
   reverse_proxy /_matrix/* http://localhost:8008
+  reverse_proxy /_synapse/client/* http://localhost:8008
 }
 
 example.com:8448 {
@@ -96,6 +105,8 @@ example.com:8448 {
     AllowEncodedSlashes NoDecode
     ProxyPass /_matrix http://127.0.0.1:8008/_matrix nocanon
     ProxyPassReverse /_matrix http://127.0.0.1:8008/_matrix
+    ProxyPass /_synapse/client http://127.0.0.1:8008/_synapse/client nocanon
+    ProxyPassReverse /_synapse/client http://127.0.0.1:8008/_synapse/client
 </VirtualHost>
 
 <VirtualHost *:8448>
@@ -119,6 +130,7 @@ frontend https
   # Matrix client traffic
   acl matrix-host hdr(host) -i matrix.example.com
   acl matrix-path path_beg /_matrix
+  acl matrix-path path_beg /_synapse/client
 
   use_backend matrix if matrix-host matrix-path
 
@@ -146,3 +158,10 @@ connecting to Synapse from a client.
 Synapse exposes a health check endpoint for use by reverse proxies.
 Each configured HTTP listener has a `/health` endpoint which always returns
 200 OK (and doesn't get logged).
+
+## Synapse administration endpoints
+
+Endpoints for administering your Synapse instance are placed under
+`/_synapse/admin`. These require authentication through an access token of an
+admin user. However as access to these endpoints grants the caller a lot of power,
+we do not recommend exposing them to the public internet without good reason.

docs/sample_config.yaml

@@ -1660,11 +1660,14 @@ trusted_key_servers:
 # At least one of `sp_config` or `config_path` must be set in this section to
 # enable SAML login.
 #
-# (You will probably also want to set the following options to `false` to
+# You will probably also want to set the following options to `false` to
 # disable the regular login/registration flows:
 #   * enable_registration
 #   * password_config.enabled
 #
+# You will also want to investigate the settings under the "sso" configuration
+# section below.
+#
 # Once SAML support is enabled, a metadata file will be exposed at
 # https://<server>:<port>/_matrix/saml2/metadata.xml, which you may be able to
 # use to configure your SAML IdP with. Alternatively, you can manually configure
@@ -1787,31 +1790,6 @@ saml2_config:
   #  - attribute: department
   #    value: "sales"
 
-  # Directory in which Synapse will try to find the template files below.
-  # If not set, default templates from within the Synapse package will be used.
-  #
-  # DO NOT UNCOMMENT THIS SETTING unless you want to customise the templates.
-  # If you *do* uncomment it, you will need to make sure that all the templates
-  # below are in the directory.
-  #
-  # Synapse will look for the following templates in this directory:
-  #
-  # * HTML page to display to users if something goes wrong during the
-  #   authentication process: 'saml_error.html'.
-  #
-  #   When rendering, this template is given the following variables:
-  #     * code: an HTML error code corresponding to the error that is being
-  #       returned (typically 400 or 500)
-  #
-  #     * msg: a textual message describing the error.
-  #
-  #   The variables will automatically be HTML-escaped.
-  #
-  # You can see the default templates at:
-  # https://github.com/matrix-org/synapse/tree/master/synapse/res/templates
-  #
-  #template_dir: "res/templates"
-
 
 # OpenID Connect integration. The following settings can be used to make Synapse
 # use an OpenID Connect Provider for authentication, instead of its internal

docs/workers.md

@@ -217,6 +217,7 @@ expressions:
     ^/_matrix/client/(api/v1|r0|unstable)/joined_groups$
     ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$
     ^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/
+    ^/_synapse/client/password_reset/email/submit_token$
 
     # Registration/login requests
     ^/_matrix/client/(api/v1|r0|unstable)/login$

mypy.ini

@@ -46,10 +46,12 @@ files =
   synapse/server_notices,
   synapse/spam_checker_api,
   synapse/state,
+  synapse/storage/databases/main/events.py,
   synapse/storage/databases/main/stream.py,
   synapse/storage/databases/main/ui_auth.py,
   synapse/storage/database.py,
   synapse/storage/engines,
+  synapse/storage/persist_events.py,
   synapse/storage/state.py,
   synapse/storage/util,
   synapse/streams,

scripts-dev/definitions.py

@@ -1,7 +1,5 @@
 #! /usr/bin/python
 
-from __future__ import print_function
-
 import argparse
 import ast
 import os

scripts-dev/dump_macaroon.py

@@ -1,7 +1,5 @@
 #!/usr/bin/env python2
 
-from __future__ import print_function
-
 import sys
 
 import pymacaroons

scripts-dev/federation_client.py

@@ -15,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import argparse
 import base64
 import json

scripts-dev/hash_history.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import sqlite3
 import sys
 

scripts/move_remote_media_to_new_store.py

@@ -32,8 +32,6 @@
     PYTHON_PATH=. ./scripts/move_remote_media_to_new_store.py <source repo> <dest repo>
 """
 
-from __future__ import print_function
-
 import argparse
 import logging
 import os

scripts/register_new_matrix_user

@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 from synapse._scripts.register_new_matrix_user import main
 
 if __name__ == "__main__":

synapse/_scripts/register_new_matrix_user.py

@@ -14,8 +14,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import argparse
 import getpass
 import hashlib

synapse/app/homeserver.py

@@ -15,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
-
 import gc
 import logging
 import math