[tooling] Upgrade psycopg, fix legacy ssl tests

monitoring/uss_qualifier/resources/interuss/datastore/datastore.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import socket
 from typing import List, Optional, Tuple
 
 import psycopg
@@ -112,6 +113,7 @@ def is_reachable(self) -> Tuple[bool, Optional[psycopg.Error]]:
             is_reachable = (
                 "password authentication failed" in err_msg
                 or "server did not complete authentication" in err_msg
+                or "server requested a hashed password" in err_msg
             )
             return is_reachable, e
         return True, None
@@ -144,16 +146,57 @@ def legacy_ssl_version_rejected(self) -> Tuple[bool, Optional[psycopg.Error]]:
         This is detected by attempting to establish a connection with the node
         forcing the client to use a TLS version < 1.2 and validating that the
         connection fails with the expected error message.
+
+        Modern libraries and Python have dropped support for TLS versions older than 1.2, as these are now considered legacy.
+
+        To be able to test those old protocols, we manually send TLS packets (captured from legacy code) and parse the result.
+        Parsing is limited, but should be good enough for our cases.
         """
-        try:
-            c = self.connect(
-                sslmode="require",
-                ssl_min_protocol_version="TLSv1",
-                ssl_max_protocol_version="TLSv1.1",
+
+        def _build_client_hello():
+            """Builds a client hello"""
+
+            return bytes.fromhex(
+                "16"  # Handshake
+                "0301"  # TLS Version: 1.0
+                "0063"  # Length
+                "01"  # Handshake type: Client hello
+                "00005f"  # Length
+                "0302"  # TLS Version: 1.1
+                "4895335bae2d2d929e34bdd5ccc89d800807bb01bbaaa7bf86efbb83a9249206"  # Random value
+                "00"  # Session ID Length
+                "0012"  # Cipher suite Length
+                "c00ac0140039c009c01300330035002f00ff"  # Cipher suites
+                "01"  # Compression method length
+                "00"  # No compression
+                "0024"  # Extentions length
+                "000b000403000102000a000c000a001d0017001e00190018002300000016000000170000"  # Extensions
             )
-            c.close()
-        except psycopg.OperationalError as e:
-            err_msg = str(e)
-            legacy_rejected = "tlsv1 alert protocol version" in err_msg
-            return legacy_rejected, e
-        return False, None
+
+        def _is_protocol_failure(data):
+            """Tests whether the server sends a protocol failure."""
+            # Format:
+            # 15     TLS Alert
+            # 03 01  TLS Version (Ignored)
+            # 00 02  Length (Ignored)
+            # 02     Level: Fatal (Ignored)
+            # 46     Description: Protocol version
+
+            content_type = data[0]
+            alert_description = data[6]
+
+            return content_type == 0x15 and alert_description == 0x46
+
+        try:
+            with socket.create_connection((self.host, self.port), timeout=5) as sock:
+                sock.sendall(bytes.fromhex("0000000804d2162f"))  # Postgres hello
+                sock.recv(16)
+                sock.sendall(_build_client_hello())
+                data = sock.recv(1024)
+
+                if not data:
+                    return False, "No response from server"
+
+                return _is_protocol_failure(data), None
+        except Exception as e:
+            return False, str(e)

monitoring/uss_qualifier/resources/interuss/datastore/datastore_test.py

@@ -0,0 +1,186 @@
+import pytest
+from testcontainers.core.container import DockerContainer
+from testcontainers.core.waiting_utils import wait_for_logs
+
+from . import DatastoreDBNode
+
+
+@pytest.fixture(scope="module")
+def good_cockroach(request):
+
+    server = DockerContainer(
+        image="cockroachdb/cockroach:v24.1.3",
+        ports=[26257],
+        command="start-single-node",
+    )
+    server.start()
+    wait_for_logs(server, "start_node_query")
+
+    return DatastoreDBNode(
+        "test", server.get_container_host_ip(), server.get_exposed_port(26257)
+    )
+
+
+@pytest.fixture(scope="module")
+def no_ssl_cockroach(request):
+
+    server = DockerContainer(
+        image="cockroachdb/cockroach:v24.1.3",
+        ports=[26257],
+        command="start-single-node --insecure",
+    )
+    server.start()
+    wait_for_logs(server, "start_node_query")
+
+    return DatastoreDBNode(
+        "test", server.get_container_host_ip(), server.get_exposed_port(26257)
+    )
+
+
+@pytest.fixture(scope="module")
+def old_ssl_cockroach(request):
+
+    server = DockerContainer(
+        image="mcuoorb/insecurecockroach:latest",
+        ports=[26257],
+        command="start-single-node",
+    )
+    server.start()
+    wait_for_logs(server, "start_node_query")
+
+    return DatastoreDBNode(
+        "test", server.get_container_host_ip(), server.get_exposed_port(26257)
+    )
+
+
+@pytest.fixture(scope="module")
+def good_yugabyte(request):
+
+    server = DockerContainer(
+        image="yugabytedb/yugabyte:2.25.2.0-b359",
+        ports=[5433],
+        command='bash -c "bin/yugabyted cert generate_server_certs --base_dir /yugabyte/certs --hostnames `hostname`  && bin/yugabyted start --secure --certs_dir=/yugabyte/certs/generated_certs/`hostname` --advertise_address=`hostname` --background=false"',
+    )
+    server.start()
+    wait_for_logs(server, "Data placement constraint successfully verified")
+
+    return DatastoreDBNode(
+        "test", server.get_container_host_ip(), server.get_exposed_port(5433)
+    )
+
+
+@pytest.fixture(scope="module")
+def no_ssl_yugabyte(request):
+
+    server = DockerContainer(
+        image="yugabytedb/yugabyte:2.25.2.0-b359",
+        ports=[5433],
+        command="bin/yugabyted start --background=false",
+    )
+    server.start()
+    wait_for_logs(server, "Data placement constraint successfully verified")
+
+    return DatastoreDBNode(
+        "test", server.get_container_host_ip(), server.get_exposed_port(5433)
+    )
+
+
+@pytest.fixture(scope="module")
+def old_ssl_yugabyte(request):
+
+    import base64
+
+    config = '{"tserver_flags": "ysql_pg_conf_csv=\\"ssl_min_protocol_version=\'TLSv1.1\'\\""}'
+    config = base64.b64encode(config.encode("utf-8"))  # Avoid escaping hell
+
+    server = DockerContainer(
+        image="yugabytedb/yugabyte:2.25.2.0-b359",
+        ports=[5433],
+        command=f"bash -c \"echo {config.decode('utf-8')} | base64 -d > /conf.conf && cat /conf.conf && bin/yugabyted cert generate_server_certs --base_dir /yugabyte/certs --hostnames `hostname` && bin/yugabyted start --secure --certs_dir=/yugabyte/certs/generated_certs/`hostname` --advertise_address=`hostname` --background=false --conf /conf.conf\"",
+    )
+    server.start()
+    wait_for_logs(server, "Data placement constraint successfully verified")
+
+    return DatastoreDBNode(
+        "test", server.get_container_host_ip(), server.get_exposed_port(5433)
+    )
+
+
+def test_datastoredbmode_connect_good_cockroach(good_cockroach):
+    is_reachable, _ = good_cockroach.is_reachable()
+    assert is_reachable
+
+
+def test_datastoredbmode_connect_good_yugabyte(good_yugabyte):
+    is_reachable, _ = good_yugabyte.is_reachable()
+    assert is_reachable
+
+
+def test_datastoredbmode_connect_no_ssl_cockroach(no_ssl_cockroach):
+    is_reachable, _ = no_ssl_cockroach.is_reachable()
+    assert is_reachable
+
+
+def test_datastoredbmode_connect_no_ssl_yugabyte(no_ssl_yugabyte):
+    is_reachable, _ = no_ssl_yugabyte.is_reachable()
+    assert is_reachable
+
+
+def test_datastoredbmode_connect_old_ssl_cockroach(old_ssl_cockroach):
+    is_reachable, _ = old_ssl_cockroach.is_reachable()
+    assert is_reachable
+
+
+def test_datastoredbmode_connect_old_ssl_yugabyte(old_ssl_yugabyte):
+    is_reachable, _ = old_ssl_yugabyte.is_reachable()
+    assert is_reachable
+
+
+def test_datastoredbmode_secure_mode_good_cockroach(good_cockroach):
+    is_secure, _ = good_cockroach.runs_in_secure_mode()
+    assert is_secure
+
+
+def test_datastoredbmode_secure_mode_good_yugabyte(good_yugabyte):
+    is_secure, _ = good_yugabyte.runs_in_secure_mode()
+    assert is_secure
+
+
+def test_datastoredbmode_secure_mode_no_ssl_cockroach(no_ssl_cockroach):
+    is_secure, _ = no_ssl_cockroach.runs_in_secure_mode()
+    assert not is_secure
+
+
+def test_datastoredbmode_secure_mode_no_ssl_yugabyte(no_ssl_yugabyte):
+    is_secure, _ = no_ssl_yugabyte.runs_in_secure_mode()
+    assert not is_secure
+
+
+def test_datastoredbmode_secure_mode_old_ssl_cockroach(old_ssl_cockroach):
+    is_secure, _ = old_ssl_cockroach.runs_in_secure_mode()
+    assert is_secure
+
+
+def test_datastoredbmode_secure_mode_old_ssl_yugabyte(old_ssl_yugabyte):
+    is_secure, _ = old_ssl_yugabyte.runs_in_secure_mode()
+    assert is_secure
+
+
+def test_datastoredbmode_reject_legacy_good_cockroach(good_cockroach):
+    legacy_rejected, _ = good_cockroach.legacy_ssl_version_rejected()
+    assert legacy_rejected
+
+
+def test_datastoredbmode_reject_legacy_good_yugabyte(good_yugabyte):
+    legacy_rejected, _ = good_yugabyte.legacy_ssl_version_rejected()
+    assert legacy_rejected
+
+
+def test_datastoredbmode_reject_legacy_old_ssl_cockroach(old_ssl_cockroach):
+    legacy_rejected, _ = old_ssl_cockroach.legacy_ssl_version_rejected()
+    assert not legacy_rejected
+
+
+def test_datastoredbmode_reject_legacy_old_ssl_yugabyte(old_ssl_yugabyte):
+    legacy_rejected, _ = old_ssl_yugabyte.legacy_ssl_version_rejected()
+    assert not legacy_rejected

monitoring/uss_qualifier/scripts/run_unit_tests.sh

@@ -23,5 +23,6 @@ docker run --name uss_qualifier_unit_test \
   --rm \
   -e MONITORING_GITHUB_ROOT=${MONITORING_GITHUB_ROOT:-} \
   -v "$(pwd):/app" \
+  -v /var/run/docker.sock:/var/run/docker.sock \
   interuss/monitoring \
   uss_qualifier/scripts/in_container/run_unit_tests.sh

pyproject.toml

@@ -45,6 +45,7 @@ dependencies = [
     "shapely",
     "structlog",
     "termcolor",
+    "testcontainers",
     "uas-standards",
     "uuid6",
 ]