From 4633a6426cf62aa87f440b2017fe7c1824847d84 Mon Sep 17 00:00:00 2001 From: Ulrich Petri Date: Mon, 19 Aug 2019 21:05:42 +0200 Subject: [PATCH] Switch to Synapse >= 1.0 (Not mergable currently because of #4634. Temporarily uses the base64 solution mentioned in the issue.) This updates our testing dependency to matrix-synapse 1.2.1. The Synapse config files generated by the integration tests have been updated to align with new / changed options. Additionally this no longer uses local TLS certificates speeding up tests. Fixes: #3387 Required for: #4292 --- raiden/network/transport/matrix/transport.py | 6 +- raiden/network/transport/matrix/utils.py | 17 ++++-- .../transport/test_matrix_transport.py | 1 + raiden/tests/unit/test_matrix_transport.py | 3 +- .../tests/utils/synapse_config.yaml.template | 55 +++++++++++++------ raiden/tests/utils/transport.py | 52 +++++++++++++++--- requirements/requirements-ci.txt | 15 ++--- requirements/requirements-dev.in | 2 +- requirements/requirements-dev.txt | 26 ++++----- 9 files changed, 119 insertions(+), 58 deletions(-) diff --git a/raiden/network/transport/matrix/transport.py b/raiden/network/transport/matrix/transport.py index 9d1c8e9b94..c7392ba215 100644 --- a/raiden/network/transport/matrix/transport.py +++ b/raiden/network/transport/matrix/transport.py @@ -488,7 +488,11 @@ def stop(self): self._client.api.session.close() self.log.debug("Matrix stopped", config=self._config) - del self.log + try: + del self.log + except AttributeError: + # During shutdown the log attribute may have already been collected + pass # parent may want to call get() after stop(), to ensure _run errors are re-raised # we don't call it here to avoid deadlock when self crashes and calls stop() on finally diff --git a/raiden/network/transport/matrix/utils.py b/raiden/network/transport/matrix/utils.py index ff2ee6abfe..ff63f3f018 100644 --- a/raiden/network/transport/matrix/utils.py +++ b/raiden/network/transport/matrix/utils.py @@ -1,3 +1,5 @@ +import binascii +import codecs import json import re from binascii import Error as DecodeError @@ -36,6 +38,7 @@ JOIN_RETRIES = 10 USERID_RE = re.compile(r"^@(0x[0-9a-f]{40})(?:\.[0-9a-f]{8})?(?::.+)?$") +DISPLAY_NAME_HEX_RE = re.compile(r"^0x[0-9a-zA-Z]{130}$") ROOM_NAME_SEPARATOR = "_" ROOM_NAME_PREFIX = "raiden" @@ -480,9 +483,10 @@ def login_or_register( else: raise ValueError("Could not register or login!") - name = encode_hex(signer.sign(client.user_id.encode())) + signature_bytes = signer.sign(client.user_id.encode()) + signature_base64 = codecs.encode(signature_bytes, "base64").strip().decode() # type: ignore user = client.get_user(client.user_id) - user.set_display_name(name) + user.set_display_name(signature_base64) log.debug( "Matrix user login", homeserver=server_name, server_url=server_url, username=username ) @@ -502,13 +506,16 @@ def validate_userid_signature(user: User) -> Optional[Address]: try: displayname = user.get_display_name() - recovered = recover( - data=user.user_id.encode(), signature=Signature(decode_hex(displayname)) - ) + if DISPLAY_NAME_HEX_RE.match(displayname): + signature_bytes = decode_hex(displayname) + else: + signature_bytes = codecs.decode(displayname.encode(), "base64") # type: ignore + recovered = recover(data=user.user_id.encode(), signature=Signature(signature_bytes)) if not (address and recovered and recovered == address): return None except ( DecodeError, + binascii.Error, TypeError, InvalidSignature, MatrixRequestError, diff --git a/raiden/tests/integration/network/transport/test_matrix_transport.py b/raiden/tests/integration/network/transport/test_matrix_transport.py index c33744ab40..84339ccf7a 100644 --- a/raiden/tests/integration/network/transport/test_matrix_transport.py +++ b/raiden/tests/integration/network/transport/test_matrix_transport.py @@ -904,6 +904,7 @@ def test_matrix_user_roaming(matrix_transports): transport0.start(raiden_service0, message_handler0, "") transport0.start_health_check(raiden_service1.address) + with Timeout(TIMEOUT_MESSAGE_RECEIVE): while not is_reachable(transport1, raiden_service0.address): gevent.sleep(0.1) diff --git a/raiden/tests/unit/test_matrix_transport.py b/raiden/tests/unit/test_matrix_transport.py index c4b48cc8b5..64168a2c04 100644 --- a/raiden/tests/unit/test_matrix_transport.py +++ b/raiden/tests/unit/test_matrix_transport.py @@ -1,3 +1,4 @@ +import codecs import random from unittest.mock import Mock, create_autospec from urllib.parse import urlparse @@ -87,7 +88,7 @@ def mock_login(user, pw, sync=True): # pylint: disable=unused-argument assert ( recover( data=client.user_id.encode(), - signature=decode_hex(user.set_display_name.call_args[0][0]), + signature=codecs.decode(user.set_display_name.call_args[0][0].encode(), "base64"), ) == signer.address ) diff --git a/raiden/tests/utils/synapse_config.yaml.template b/raiden/tests/utils/synapse_config.yaml.template index a5b85c5d14..92bf985f87 100644 --- a/raiden/tests/utils/synapse_config.yaml.template +++ b/raiden/tests/utils/synapse_config.yaml.template @@ -1,24 +1,23 @@ -tls_certificate_path: "{server_dir}/localhost:{port}.tls.crt" -tls_private_key_path: "{server_dir}/localhost:{port}.tls.key" -tls_dh_params_path: "{server_dir}/localhost:{port}.tls.dh" - -no_tls: false +no_tls: true +federation_verify_certificates: false +accept_keys_insecurely: true tls_fingerprints: [] +trusted_key_servers: [] + server_name: "localhost:{port}" -web_client: true soft_file_limit: 0 listeners: - port: {port} - tls: true + tls: false bind_addresses: ['127.0.0.1'] type: http x_forwarded: false resources: - - names: [client, webclient] # changed + - names: [client] # changed compress: true - names: [federation] compress: false @@ -28,16 +27,37 @@ database: args: database: ":memory:" # changed -event_cache_size: "10K" +user_directory: + enabled: true + search_all_users: true -rc_messages_per_second: 200000 # changed -rc_message_burst_count: 100000 # changed +event_cache_size: "10K" -federation_rc_window_size: 1000 -federation_rc_sleep_limit: 10 -federation_rc_sleep_delay: 500 -federation_rc_reject_limit: 50 -federation_rc_concurrent: 3 +rc_message: + per_second: 50000 # changed + burst_count: 100000 # changed + +rc_registration: + per_second: 0.17 + burst_count: 3 + +rc_login: + address: + per_second: 100 + burst_count: 100 + account: + per_second: 100 + burst_count: 100 + failed_attempts: + per_second: 100 + burst_count: 100 + +rc_federation: + window_size: 1000 + sleep_limit: 10 + sleep_delay: 500 + reject_limit: 50 + concurrent: 3 max_upload_size: "0M" # changed max_image_pixels: "0M" # changed @@ -70,6 +90,9 @@ password_providers: - module: 'raiden.tests.utils.transport.EthAuthProvider' config: enabled: true + - module: 'raiden.tests.utils.transport.NoTLSFederationMonkeyPatchProvider' + config: + enabled: true # Whether to allow non server admins to create groups on this server enable_group_creation: false diff --git a/raiden/tests/utils/transport.py b/raiden/tests/utils/transport.py index 13dc94cd59..0b03845134 100644 --- a/raiden/tests/utils/transport.py +++ b/raiden/tests/utils/transport.py @@ -109,6 +109,39 @@ def parse_config(config): return config +# Used from within synapse during tests +class NoTLSFederationMonkeyPatchProvider: + """ Dummy auth provider that disables TLS on S2S federation. + + This is used by the integration tests to avoid the need for tls certificates. + It's implemented as an auth provider since that's a handy way to inject code into the + synapse process. + + It works by replacing ``synapse.crypto.context_factory.ClientTLSOptionsFactory`` with an + object that returns ``None`` when instantiated. + """ + + __version__ = "0.1" + + class NoTLSFactory: + def __new__(cls, *args, **kwargs): # pylint: disable=unused-argument + return None + + def __init__(self, config, account_handler): # pylint: disable=unused-argument + pass + + @defer.inlineCallbacks + def check_password(self, user_id, password): # pylint: disable=unused-argument,no-self-use + defer.returnValue(False) + + @staticmethod + def parse_config(config): + from synapse.crypto import context_factory + + context_factory.ClientTLSOptionsFactory = NoTLSFederationMonkeyPatchProvider.NoTLSFactory + return config + + def make_requests_insecure(): """ Prevent `requests` from performing TLS verification. @@ -179,9 +212,17 @@ def matrix_server_starter( server_urls: List[ParsedURL] = [] for _, port in zip(range(count), free_port_generator): server_name, config_file = config_generator(port) - server_url = ParsedURL(f"https://{server_name}") + server_url = ParsedURL(f"http://{server_name}") server_urls.append(server_url) + synapse_cmd = [ + sys.executable, + "-m", + "synapse.app.homeserver", + f"--server-name={server_name}", + f"--config-path={config_file!s}", + ] + synapse_io: EXECUTOR_IO = DEVNULL # Used in CI to capture the logs for failure analysis if _SYNAPSE_LOGS_PATH is not None: @@ -195,6 +236,7 @@ def matrix_server_starter( header = f"{header}: {log_context}" header = f" {header} " log_file.write(f"{header:=^100}\n") + log_file.write(f"Cmd: `{' '.join(synapse_cmd)}`\n") log_file.flush() synapse_io = DEVNULL, log_file, STDOUT @@ -203,13 +245,7 @@ def matrix_server_starter( sleep = 0.1 executor = HTTPExecutor( - [ - sys.executable, - "-m", - "synapse.app.homeserver", - f"--server-name={server_name}", - f"--config-path={config_file!s}", - ], + synapse_cmd, url=urljoin(server_url, "/_matrix/client/versions"), method="GET", timeout=startup_timeout, diff --git a/requirements/requirements-ci.txt b/requirements/requirements-ci.txt index 695295def4..ead730e795 100644 --- a/requirements/requirements-ci.txt +++ b/requirements/requirements-ci.txt @@ -37,7 +37,6 @@ cryptography==2.7 cytoolz==0.9.0.1 daemonize==2.5.0 decorator==4.4.0 -defusedxml==0.6.0 docutils==0.14 entrypoints==0.3 eth-abi==1.3.0 @@ -76,21 +75,18 @@ itsdangerous==1.1.0 jinja2==2.10.1 jsonschema==3.0.1 lazy-object-proxy==1.4.1 -ldap3==2.6 lru-dict==1.1.6 macholib==1.11 # via pyinstaller markupsafe==1.1.1 marshmallow-dataclass==6.0.0rc4 marshmallow-polyfield==5.7 marshmallow==3.0.0rc8 -matrix-angular-sdk==0.6.8 matrix-client==0.3.2 -matrix-synapse-ldap3==0.1.3 -matrix-synapse==0.33.9 +matrix-synapse==1.2.1 mccabe==0.6.1 mirakuru==1.1.0 more-itertools==7.0.0 -msgpack-python==0.5.6 +msgpack==0.6.1 mypy-extensions==0.4.1 mypy==0.720 netaddr==0.7.19 @@ -124,19 +120,18 @@ pygments==2.4.2 pyhamcrest==1.9.0 pyinstaller==3.4 pylint==2.3.1 -pymacaroons-pynacl==0.9.3 +pymacaroons==0.13.0 pynacl==1.3.0 pyopenssl==19.0.0 pyparsing==2.4.0 pyrsistent==0.15.2 -pysaml2==4.7.0 pysha3==1.0.2 pytest-forked==1.0.2 pytest-random==0.2 pytest-select==0.1.2 pytest-xdist==1.28.0 pytest==4.6.3 -python-dateutil==2.8.0 +python-dateutil==2.8.0 # via s3cmd python-magic==0.4.15 # via s3cmd pytoml==0.1.20 pytz==2019.1 @@ -187,4 +182,4 @@ zipp==0.5.1 zope.interface==4.6.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.0.1 # via ipython, jsonschema, pyhamcrest, pyinstaller, sphinx, zope.interface +# setuptools==41.1.0 # via ipython, jsonschema, pyhamcrest, pyinstaller, sphinx, zope.interface diff --git a/requirements/requirements-dev.in b/requirements/requirements-dev.in index ae6b848c02..6c24ea754a 100644 --- a/requirements/requirements-dev.in +++ b/requirements/requirements-dev.in @@ -40,7 +40,7 @@ coverage bump2version # Test support -matrix-synapse==0.33.9 +matrix-synapse # Pin psycopg2 to prevent having to compile the c-extension # (see https://github.com/raiden-network/raiden/issues/3745) diff --git a/requirements/requirements-dev.txt b/requirements/requirements-dev.txt index f7c9dbec39..2df2264c1e 100644 --- a/requirements/requirements-dev.txt +++ b/requirements/requirements-dev.txt @@ -32,11 +32,10 @@ colorama==0.4.1 colour==0.1.5 constantly==15.1.0 # via twisted coverage==4.5.3 -cryptography==2.7 # via pyopenssl, pysaml2, service-identity +cryptography==2.7 # via pyopenssl, service-identity cytoolz==0.9.0.1 daemonize==2.5.0 # via matrix-synapse decorator==4.4.0 -defusedxml==0.6.0 # via pysaml2 docutils==0.14 entrypoints==0.3 # via flake8 eth-abi==1.3.0 @@ -74,20 +73,17 @@ itsdangerous==1.1.0 jinja2==2.10.1 jsonschema==3.0.1 # via matrix-synapse lazy-object-proxy==1.4.1 # via astroid -ldap3==2.6 # via matrix-synapse-ldap3 lru-dict==1.1.6 markupsafe==1.1.1 marshmallow-dataclass==6.0.0rc4 marshmallow-polyfield==5.7 marshmallow==3.0.0rc8 -matrix-angular-sdk==0.6.8 # via matrix-synapse matrix-client==0.3.2 -matrix-synapse-ldap3==0.1.3 # via matrix-synapse -matrix-synapse==0.33.9 +matrix-synapse==1.2.1 mccabe==0.6.1 # via flake8, pylint mirakuru==1.1.0 more-itertools==7.0.0 # via pytest -msgpack-python==0.5.6 # via matrix-synapse +msgpack==0.6.1 # via matrix-synapse mypy-extensions==0.4.1 mypy==0.720 netaddr==0.7.19 # via matrix-synapse @@ -111,7 +107,7 @@ py-geth==2.1.0 py-solc==3.2.0 py==1.8.0 # via pytest pyasn1-modules==0.2.5 # via matrix-synapse, service-identity -pyasn1==0.4.5 # via ldap3, matrix-synapse, pyasn1-modules, service-identity +pyasn1==0.4.5 # via matrix-synapse, pyasn1-modules, service-identity pycodestyle==2.5.0 # via autopep8, flake8 pycparser==2.19 pycryptodome==3.8.2 @@ -119,19 +115,17 @@ pyflakes==2.1.1 # via flake8 pygments==2.4.2 pyhamcrest==1.9.0 # via twisted pylint==2.3.1 -pymacaroons-pynacl==0.9.3 # via matrix-synapse -pynacl==1.3.0 # via matrix-synapse, pymacaroons-pynacl, signedjson -pyopenssl==19.0.0 # via matrix-synapse, pysaml2, twisted +pymacaroons==0.13.0 # via matrix-synapse +pynacl==1.3.0 # via matrix-synapse, pymacaroons, signedjson +pyopenssl==19.0.0 # via matrix-synapse, twisted pyparsing==2.4.0 pyrsistent==0.15.2 # via jsonschema -pysaml2==4.7.0 # via matrix-synapse pysha3==1.0.2 pytest-forked==1.0.2 # via pytest-xdist pytest-random==0.2 pytest-select==0.1.2 pytest-xdist==1.28.0 pytest==4.6.3 -python-dateutil==2.8.0 # via pysaml2 pytoml==0.1.20 pytz==2019.1 pyyaml==5.1.1 # via matrix-synapse @@ -143,7 +137,7 @@ requests==2.22.0 rlp==1.1.0 semantic-version==2.6.0 semver==2.8.1 -service-identity==18.1.0 # via matrix-synapse, matrix-synapse-ldap3, twisted +service-identity==18.1.0 # via matrix-synapse, twisted signedjson==1.0.0 # via matrix-synapse simplegeneric==0.8.1 simplejson==3.16.0 # via canonicaljson @@ -161,7 +155,7 @@ toml==0.10.0 # via black toolz==0.9.0 traitlets==4.3.2 treq==18.6.0 # via matrix-synapse -twisted[tls]==19.2.1 # via matrix-synapse, matrix-synapse-ldap3, treq +twisted[tls]==19.2.1 # via matrix-synapse, treq typed-ast==1.4.0 # via astroid, mypy typing-extensions==3.7.4 typing-inspect==0.4.0 @@ -180,4 +174,4 @@ zipp==0.5.1 # via importlib-metadata zope.interface==4.6.0 # via twisted # The following packages are considered to be unsafe in a requirements file: -# setuptools==41.0.1 # via ipython, jsonschema, pyhamcrest, sphinx, zope.interface +# setuptools==41.1.0 # via ipython, jsonschema, pyhamcrest, sphinx, zope.interface