test: Ability to test multiple relay versions in one chain (#1028)

Add second CI target that runs older Relays in combination with newer ones. This is supposed to test general compatibility between Relay versions.

See getsentry.atlassian.net/browse/INGEST-71 for more details.

Some tests have been added in later Relay versions. Those tests need to specify min_relay_version such that we don't run a newer regression test against an older Relay version, or a feature test against a Relay version where the feature is not implemented.

Relay binaries are used instead of Docker because Docker for Mac is annoying to configure for networking.

.github/workflows/ci.yml

@@ -229,3 +229,5 @@ jobs:
 
       - name: Run tests
         run: pytest tests -n auto -v
+        env:
+          RELAY_VERSION_CHAIN: '20.6.0,latest'

tests/integration/conftest.py

@@ -1,5 +1,7 @@
 import socket
 import subprocess
+import itertools
+import os
 from os import path
 from typing import Optional
 import json
@@ -10,7 +12,7 @@
 from .fixtures.gobetween import gobetween  # noqa
 from .fixtures.haproxy import haproxy  # noqa
 from .fixtures.mini_sentry import mini_sentry  # noqa
-from .fixtures.relay import relay  # noqa
+from .fixtures.relay import relay, get_relay_binary  # noqa
 from .fixtures.processing import (
     kafka_consumer,
     get_topic_name,
@@ -61,24 +63,80 @@ def inner(name):
     return inner
 
 
-@pytest.fixture(  # noqa
-    params=[
+def _parse_version(version):
+    if version == "latest":
+        return (float("inf"),)
+
+    return tuple(map(int, version.split(".")))
+
+
+def _get_relay_chains():
+    rv = [
         "relay->sentry",
         "relay->relay->sentry",
         "relay->ha->relay->gb->sentry",
         "relay->gb->relay->ha->sentry",
-    ],
-)
+    ]
+
+    configured_versions = list(
+        filter(bool, (os.environ.get("RELAY_VERSION_CHAIN") or "").split(","))
+    )
+
+    if configured_versions:
+        rv.append(
+            "->".join(f"relay(version='{version}')" for version in configured_versions)
+            + "->sentry"
+        )
+
+    return rv
+
+
+@pytest.fixture(params=_get_relay_chains())
 def relay_chain(request, mini_sentry, relay, gobetween, haproxy):  # noqa
-    parts = iter(reversed(request.param.split("->")))
-    assert next(parts) == "sentry"
+    """
+    test fixture that nests relays with mini_sentry and some random proxy
+    services. Is parametrized using strings following this syntax:
 
-    factories = {"relay": relay, "gb": gobetween, "ha": haproxy}
+        service1->service2->sentry
+        service1(arg=1, arg2=2)->service2->sentry
+
+    Which arguments are accepted by a
+    service depends on the service type:
+
+    - `gb` to configure and start gobetween. `haproxy` to configure and start
+      HAProxy. Those two are not run in CI, they used to be included because we
+      had transport-layer problems with actix-web client that were only
+      reproducible by adding more proxies.
+
+    - `relay` to spawn relay.
+
+    - `sentry` to start mini_sentry. Must be the last service and cannot appear
+      before.
+    """
+
+    def inner(min_relay_version="0"):
+        def relay_with_version(upstream, version="latest", **kwargs):
+            if _parse_version(version) < _parse_version(min_relay_version):
+                version = min_relay_version
+
+            return relay(upstream, version=version, **kwargs)
+
+        factories = {"gb": gobetween, "ha": haproxy, "relay": relay_with_version}
+
+        parts = iter(reversed(request.param.split("->")))
+        assert next(parts) == "sentry"
 
-    def inner():
         rv = mini_sentry
         for part in parts:
-            rv = factories[part](rv)
+            if "(" in part and part.endswith(")"):
+                service_name, service_args = part.rstrip(")").split("(")
+                service_args = eval(f"dict({service_args})")
+            else:
+                service_name = part
+                service_args = {}
+
+            rv = factories[service_name](rv, **service_args)
+
         return rv
 
     return inner

tests/integration/fixtures/mini_sentry.py

@@ -21,6 +21,13 @@
     CURRENT_VERSION = _version_re.search(f.read()).group(1)
 
 
+def _parse_version(version):
+    if version == "latest":
+        return (float("inf"),)
+
+    return tuple(map(int, version.split(".")))
+
+
 class Sentry(SentryLike):
     def __init__(self, server_address, app):
         super(Sentry, self).__init__(server_address)
@@ -193,17 +200,27 @@ def count_hits():
     @app.route("/api/0/relays/register/challenge/", methods=["POST"])
     def get_challenge():
         relay_id = flask_request.json["relay_id"]
-        public_key = flask_request.json["public_key"]
-        version = flask_request.json["version"]
-
         assert relay_id == flask_request.headers["x-sentry-relay-id"]
+
         if relay_id not in sentry.known_relays:
             abort(403, "unknown relay")
 
-        if version != CURRENT_VERSION:
+        relay_info = sentry.known_relays[relay_id]
+
+        public_key = flask_request.json["public_key"]
+        version = flask_request.json.get("version")
+        registered_version = relay_info["version"]
+
+        if version is None:
+            if _parse_version(registered_version) >= (20, 8):
+                abort(400, "missing version")
+
+        elif version != registered_version and not (
+            registered_version == "latest" and version == CURRENT_VERSION
+        ):
             abort(403, "outdated version")
 
-        authenticated_relays[relay_id] = sentry.known_relays[relay_id]
+        authenticated_relays[relay_id] = relay_info
         return jsonify({"token": "123", "relay_id": relay_id})
 
     @app.route("/api/0/relays/register/response/", methods=["POST"])

tests/integration/fixtures/relay.py

@@ -1,6 +1,9 @@
 import json
 import os
+import sys
 import signal
+import stat
+import requests
 import subprocess
 
 import pytest
@@ -24,6 +27,7 @@ def __init__(
         relay_id,
         config_dir,
         options,
+        version,
     ):
         super(Relay, self).__init__(server_address, upstream, public_key)
 
@@ -32,6 +36,7 @@ def __init__(
         self.secret_key = secret_key
         self.config_dir = config_dir
         self.options = options
+        self.version = version
 
     def shutdown(self, sig=signal.SIGKILL):
         self.process.send_signal(sig)
@@ -44,15 +49,58 @@ def shutdown(self, sig=signal.SIGKILL):
 
 
 @pytest.fixture
-def relay(mini_sentry, random_port, background_process, config_dir):
+def get_relay_binary():
+    def inner(version="latest"):
+        if version == "latest":
+            return RELAY_BIN
+
+        if sys.platform == "linux" or sys.platform == "linux2":
+            filename = "relay-Linux-x86_64"
+        elif sys.platform == "darwin":
+            filename = "relay-Darwin-x86_64"
+        elif sys.platform == "win32":
+            filename = "relay-Windows-x86_64.exe"
+
+        download_path = f"target/relay_releases_cache/{filename}_{version}"
+
+        if not os.path.exists(download_path):
+            download_url = (
+                f"https://github.com/getsentry/relay/releases/download/"
+                f"{version}/{filename}"
+            )
+
+            headers = {}
+            if "GITHUB_TOKEN" in os.environ:
+                headers["Authorization"] = f"Bearer {os.environ['GITHUB_TOKEN']}"
+
+            os.makedirs(os.path.dirname(download_path), exist_ok=True)
+
+            with requests.get(download_url, headers=headers) as r:
+                r.raise_for_status()
+
+                with open(download_path, "wb") as f:
+                    for chunk in r.iter_content(chunk_size=8192):
+                        f.write(chunk)
+
+        os.chmod(download_path, 0o700 | stat.S_IEXEC)
+
+        return [download_path]
+
+    return inner
+
+
+@pytest.fixture
+def relay(mini_sentry, random_port, background_process, config_dir, get_relay_binary):
     def inner(
         upstream,
         options=None,
         prepare=None,
         external=None,
         wait_healthcheck=True,
         static_relays=None,
+        version="latest",
     ):
+        relay_bin = get_relay_binary(version)
         host = "127.0.0.1"
         port = random_port()
 
@@ -84,7 +132,7 @@ def inner(
         dir.join("config.yml").write(json.dumps(default_opts))
 
         output = subprocess.check_output(
-            RELAY_BIN + ["-c", str(dir), "credentials", "generate"]
+            relay_bin + ["-c", str(dir), "credentials", "generate"]
         )
 
         # now that we have generated a credentials file get the details
@@ -103,9 +151,10 @@ def inner(
         mini_sentry.known_relays[relay_id] = {
             "publicKey": public_key,
             "internal": not external,
+            "version": version,
         }
 
-        process = background_process(RELAY_BIN + ["-c", str(dir), "run"])
+        process = background_process(relay_bin + ["-c", str(dir), "run"])
 
         relay = Relay(
             (host, port),
@@ -116,6 +165,7 @@ def inner(
             relay_id,
             dir,
             default_opts,
+            version,
         )
 
         if wait_healthcheck:

tests/integration/test_envelope.py

@@ -117,7 +117,7 @@ def test_normalize_measurement_interface(
 
 
 def test_empty_measurement_interface(mini_sentry, relay_chain):
-    relay = relay_chain()
+    relay = relay_chain(min_relay_version="20.10.0")
     mini_sentry.add_basic_project_config(42)
 
     transaction_item = generate_transaction_item()
@@ -282,7 +282,7 @@ def test_ops_breakdowns(mini_sentry, relay_with_processing, transactions_consume
 
 
 def test_sample_rates(mini_sentry, relay_chain):
-    relay = relay_chain()
+    relay = relay_chain(min_relay_version="21.1.0")
     mini_sentry.add_basic_project_config(42)
 
     sample_rates = [

tests/integration/test_forwarding.py

@@ -38,7 +38,7 @@ def test():
 
         return Response(_data, headers=headers)
 
-    relay = relay_chain()
+    relay = relay_chain(min_relay_version="21.6.1")
 
     headers = {"Content-Type": "application/octet-stream"}
 

tests/integration/test_store.py

@@ -943,7 +943,7 @@ def is_live():
 
 
 def test_store_invalid_gzip(mini_sentry, relay_chain):
-    relay = relay_chain()
+    relay = relay_chain(min_relay_version="21.6.0")
     project_id = 42
     mini_sentry.add_basic_project_config(project_id)