Switch to Synapse >= 1.0

(Not mergable currently because of raiden-network#4634. Temporarily uses the base64
solution mentioned in the issue.)

This updates our testing dependency to matrix-synapse 1.2.1.

The Synapse config files generated by the integration tests have been
updated to align with new / changed options.

Additionally this no longer uses local TLS certificates speeding up
tests.

Fixes: raiden-network#3387
Required for: raiden-network#4292

raiden/network/transport/matrix/transport.py

@@ -488,7 +488,11 @@ def stop(self):
         self._client.api.session.close()
 
         self.log.debug("Matrix stopped", config=self._config)
-        del self.log
+        try:
+            del self.log
+        except AttributeError:
+            # During shutdown the log attribute may have already been collected
+            pass
         # parent may want to call get() after stop(), to ensure _run errors are re-raised
         # we don't call it here to avoid deadlock when self crashes and calls stop() on finally
 

raiden/network/transport/matrix/utils.py

@@ -1,3 +1,5 @@
+import binascii
+import codecs
 import json
 import re
 from binascii import Error as DecodeError
@@ -36,6 +38,7 @@
 
 JOIN_RETRIES = 10
 USERID_RE = re.compile(r"^@(0x[0-9a-f]{40})(?:\.[0-9a-f]{8})?(?::.+)?$")
+DISPLAY_NAME_HEX_RE = re.compile(r"^0x[0-9a-zA-Z]{130}$")
 ROOM_NAME_SEPARATOR = "_"
 ROOM_NAME_PREFIX = "raiden"
 
@@ -480,9 +483,10 @@ def login_or_register(
     else:
         raise ValueError("Could not register or login!")
 
-    name = encode_hex(signer.sign(client.user_id.encode()))
+    signature_bytes = signer.sign(client.user_id.encode())
+    signature_base64 = codecs.encode(signature_bytes, "base64").strip().decode()  # type: ignore
     user = client.get_user(client.user_id)
-    user.set_display_name(name)
+    user.set_display_name(signature_base64)
     log.debug(
         "Matrix user login", homeserver=server_name, server_url=server_url, username=username
     )
@@ -502,13 +506,16 @@ def validate_userid_signature(user: User) -> Optional[Address]:
 
     try:
         displayname = user.get_display_name()
-        recovered = recover(
-            data=user.user_id.encode(), signature=Signature(decode_hex(displayname))
-        )
+        if DISPLAY_NAME_HEX_RE.match(displayname):
+            signature_bytes = decode_hex(displayname)
+        else:
+            signature_bytes = codecs.decode(displayname.encode(), "base64")  # type: ignore
+        recovered = recover(data=user.user_id.encode(), signature=Signature(signature_bytes))
         if not (address and recovered and recovered == address):
             return None
     except (
         DecodeError,
+        binascii.Error,
         TypeError,
         InvalidSignature,
         MatrixRequestError,

raiden/tests/integration/network/transport/test_matrix_transport.py

@@ -904,6 +904,7 @@ def test_matrix_user_roaming(matrix_transports):
 
     transport0.start(raiden_service0, message_handler0, "")
     transport0.start_health_check(raiden_service1.address)
+
     with Timeout(TIMEOUT_MESSAGE_RECEIVE):
         while not is_reachable(transport1, raiden_service0.address):
             gevent.sleep(0.1)

raiden/tests/unit/test_matrix_transport.py

@@ -1,3 +1,4 @@
+import codecs
 import random
 from unittest.mock import Mock, create_autospec
 from urllib.parse import urlparse
@@ -87,7 +88,7 @@ def mock_login(user, pw, sync=True):  # pylint: disable=unused-argument
     assert (
         recover(
             data=client.user_id.encode(),
-            signature=decode_hex(user.set_display_name.call_args[0][0]),
+            signature=codecs.decode(user.set_display_name.call_args[0][0].encode(), "base64"),
         )
         == signer.address
     )

raiden/tests/utils/synapse_config.yaml.template

@@ -1,24 +1,23 @@
-tls_certificate_path: "{server_dir}/localhost:{port}.tls.crt"
-tls_private_key_path: "{server_dir}/localhost:{port}.tls.key"
-tls_dh_params_path: "{server_dir}/localhost:{port}.tls.dh"
-
-no_tls: false
+no_tls: true
+federation_verify_certificates: false
+accept_keys_insecurely: true
 tls_fingerprints: []
+trusted_key_servers: []
+
 server_name: "localhost:{port}"
-web_client: true
 
 soft_file_limit: 0
 
 listeners:
   - port: {port}
-    tls: true
+    tls: false
     bind_addresses: ['127.0.0.1']
     type: http
 
     x_forwarded: false
 
     resources:
-      - names: [client, webclient]  # changed
+      - names: [client]  # changed
         compress: true
       - names: [federation]
         compress: false
@@ -28,16 +27,37 @@ database:
   args:
     database: ":memory:"  # changed
 
-event_cache_size: "10K"
+user_directory:
+  enabled: true
+  search_all_users: true
 
-rc_messages_per_second: 200000  # changed
-rc_message_burst_count: 100000  # changed
+event_cache_size: "10K"
 
-federation_rc_window_size: 1000
-federation_rc_sleep_limit: 10
-federation_rc_sleep_delay: 500
-federation_rc_reject_limit: 50
-federation_rc_concurrent: 3
+rc_message:
+  per_second:   50000  # changed
+  burst_count: 100000  # changed
+
+rc_registration:
+  per_second: 0.17
+  burst_count: 3
+
+rc_login:
+  address:
+    per_second: 100
+    burst_count: 100
+  account:
+    per_second: 100
+    burst_count: 100
+  failed_attempts:
+    per_second: 100
+    burst_count: 100
+
+rc_federation:
+  window_size: 1000
+  sleep_limit: 10
+  sleep_delay: 500
+  reject_limit: 50
+  concurrent: 3
 
 max_upload_size: "0M"  # changed
 max_image_pixels: "0M"  # changed
@@ -70,6 +90,9 @@ password_providers:
   - module: 'raiden.tests.utils.transport.EthAuthProvider'
     config:
       enabled: true
+  - module: 'raiden.tests.utils.transport.NoTLSFederationMonkeyPatchProvider'
+    config:
+      enabled: true
 
 # Whether to allow non server admins to create groups on this server
 enable_group_creation: false

raiden/tests/utils/transport.py

@@ -109,6 +109,39 @@ def parse_config(config):
         return config
 
 
+# Used from within synapse during tests
+class NoTLSFederationMonkeyPatchProvider:
+    """ Dummy auth provider that disables TLS on S2S federation.
+
+    This is used by the integration tests to avoid the need for tls certificates.
+    It's implemented as an auth provider since that's a handy way to inject code into the
+    synapse process.
+
+    It works by replacing ``synapse.crypto.context_factory.ClientTLSOptionsFactory`` with an
+    object that returns ``None`` when instantiated.
+    """
+
+    __version__ = "0.1"
+
+    class NoTLSFactory:
+        def __new__(cls, *args, **kwargs):  # pylint: disable=unused-argument
+            return None
+
+    def __init__(self, config, account_handler):  # pylint: disable=unused-argument
+        pass
+
+    @defer.inlineCallbacks
+    def check_password(self, user_id, password):  # pylint: disable=unused-argument,no-self-use
+        defer.returnValue(False)
+
+    @staticmethod
+    def parse_config(config):
+        from synapse.crypto import context_factory
+
+        context_factory.ClientTLSOptionsFactory = NoTLSFederationMonkeyPatchProvider.NoTLSFactory
+        return config
+
+
 def make_requests_insecure():
     """
     Prevent `requests` from performing TLS verification.
@@ -179,9 +212,17 @@ def matrix_server_starter(
         server_urls: List[ParsedURL] = []
         for _, port in zip(range(count), free_port_generator):
             server_name, config_file = config_generator(port)
-            server_url = ParsedURL(f"https://{server_name}")
+            server_url = ParsedURL(f"http://{server_name}")
             server_urls.append(server_url)
 
+            synapse_cmd = [
+                sys.executable,
+                "-m",
+                "synapse.app.homeserver",
+                f"--server-name={server_name}",
+                f"--config-path={config_file!s}",
+            ]
+
             synapse_io: EXECUTOR_IO = DEVNULL
             # Used in CI to capture the logs for failure analysis
             if _SYNAPSE_LOGS_PATH is not None:
@@ -195,6 +236,7 @@ def matrix_server_starter(
                     header = f"{header}: {log_context}"
                 header = f" {header} "
                 log_file.write(f"{header:=^100}\n")
+                log_file.write(f"Cmd: `{' '.join(synapse_cmd)}`\n")
                 log_file.flush()
 
                 synapse_io = DEVNULL, log_file, STDOUT
@@ -203,13 +245,7 @@ def matrix_server_starter(
             sleep = 0.1
 
             executor = HTTPExecutor(
-                [
-                    sys.executable,
-                    "-m",
-                    "synapse.app.homeserver",
-                    f"--server-name={server_name}",
-                    f"--config-path={config_file!s}",
-                ],
+                synapse_cmd,
                 url=urljoin(server_url, "/_matrix/client/versions"),
                 method="GET",
                 timeout=startup_timeout,

requirements/requirements-ci.txt

@@ -37,7 +37,6 @@ cryptography==2.7
 cytoolz==0.9.0.1
 daemonize==2.5.0
 decorator==4.4.0
-defusedxml==0.6.0
 docutils==0.14
 entrypoints==0.3
 eth-abi==1.3.0
@@ -76,21 +75,18 @@ itsdangerous==1.1.0
 jinja2==2.10.1
 jsonschema==3.0.1
 lazy-object-proxy==1.4.1
-ldap3==2.6
 lru-dict==1.1.6
 macholib==1.11            # via pyinstaller
 markupsafe==1.1.1
 marshmallow-dataclass==6.0.0rc4
 marshmallow-polyfield==5.7
 marshmallow==3.0.0rc8
-matrix-angular-sdk==0.6.8
 matrix-client==0.3.2
-matrix-synapse-ldap3==0.1.3
-matrix-synapse==0.33.9
+matrix-synapse==1.2.1
 mccabe==0.6.1
 mirakuru==1.1.0
 more-itertools==7.0.0
-msgpack-python==0.5.6
+msgpack==0.6.1
 mypy-extensions==0.4.1
 mypy==0.720
 netaddr==0.7.19
@@ -124,19 +120,18 @@ pygments==2.4.2
 pyhamcrest==1.9.0
 pyinstaller==3.4
 pylint==2.3.1
-pymacaroons-pynacl==0.9.3
+pymacaroons==0.13.0
 pynacl==1.3.0
 pyopenssl==19.0.0
 pyparsing==2.4.0
 pyrsistent==0.15.2
-pysaml2==4.7.0
 pysha3==1.0.2
 pytest-forked==1.0.2
 pytest-random==0.2
 pytest-select==0.1.2
 pytest-xdist==1.28.0
 pytest==4.6.3
-python-dateutil==2.8.0
+python-dateutil==2.8.0    # via s3cmd
 python-magic==0.4.15      # via s3cmd
 pytoml==0.1.20
 pytz==2019.1
@@ -187,4 +182,4 @@ zipp==0.5.1
 zope.interface==4.6.0
 
 # The following packages are considered to be unsafe in a requirements file:
-# setuptools==41.0.1        # via ipython, jsonschema, pyhamcrest, pyinstaller, sphinx, zope.interface
+# setuptools==41.1.0        # via ipython, jsonschema, pyhamcrest, pyinstaller, sphinx, zope.interface

requirements/requirements-dev.in

@@ -40,7 +40,7 @@ coverage
 bump2version
 
 # Test support
-matrix-synapse==0.33.9
+matrix-synapse
 
 # Pin psycopg2 to prevent having to compile the c-extension
 # (see https://github.com/raiden-network/raiden/issues/3745)