Add TLS on Keycloak server (#3427)

PBENCH-1138 1. Run Keycloak containers with SSL support 2. Make necessary changes in the pbench-server to connect to the Keycloak running on SSL. 3. Note: run-pbench-in-a-can script creates self signed pbench-server certificate and the same certificate is used for Keycloak configuration.
distributed-system-analysis · Jul 3, 2023 · 3b8bf2f · 3b8bf2f
1 parent 775da21
commit 3b8bf2f
Show file tree

Hide file tree

Showing 14 changed files with 81 additions and 46 deletions.
diff --git a/exec-tests b/exec-tests
@@ -243,7 +243,7 @@ if [[ "${major}" == "all" || "${major}" == "server" ]]; then
         shift
         posargs="${@}"
         # We use SQLALCHEMY_SILENCE_UBER_WARNING here ... (see above).
-        SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True PBENCH_SERVER=${server_arg} KEEP_DATASETS="${keep_datasets}" pytest --tb=native -v -s -rs --pyargs ${posargs} pbench.test.functional.server
+        REQUESTS_CA_BUNDLE=${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt SQLALCHEMY_SILENCE_UBER_WARNING=1 PYTHONUNBUFFERED=True PBENCH_SERVER=${server_arg} KEEP_DATASETS="${keep_datasets}" pytest --tb=native -v -s -rs --pyargs ${posargs} pbench.test.functional.server
         rc=${?}
     fi
 fi

diff --git a/jenkins/run-server-func-tests b/jenkins/run-server-func-tests
@@ -8,15 +8,7 @@ export PB_SERVER_IMAGE_TAG=${PB_SERVER_IMAGE_TAG:-"$(cat jenkins/branch.name)"}
 export PB_POD_NAME=${PB_POD_NAME:-"pbench-in-a-can_${PB_SERVER_IMAGE_TAG}"}
 export PB_SERVER_CONTAINER_NAME=${PB_SERVER_CONTAINER_NAME:-"${PB_POD_NAME}-pbenchserver"}
 
-# Note:  the value of PB_HOST_IP will be used to generate the TLS certificate
-# and so it (not `localhost`) must also be used to access the Pbench Server;
-# otherwise, the TLS validation will fail due to a host mismatch.
-if [[ -z "${PB_HOST_IP}" ]]; then
-    host_ip_list=$(hostname -I)
-    PB_HOST_IP=${host_ip_list%% *}
-    export PB_HOST_IP
-fi
-SERVER_URL="https://${PB_HOST_IP}:8443"
+SERVER_URL="https://localhost:8443"
 SERVER_API_ENDPOINTS="${SERVER_URL}/api/v1/endpoints"
 
 # Have Curl use the Pbench CA certificate to validate the TLS/SSL connection

diff --git a/lib/pbench/client/__init__.py b/lib/pbench/client/__init__.py
@@ -1,5 +1,4 @@
 from enum import Enum
-import os
 from pathlib import Path
 from typing import Iterator, Optional
 from urllib import parse
@@ -319,9 +318,6 @@ def connect(self, headers: Optional[dict[str, str]] = None) -> None:
         url = parse.urljoin(self.url, "api/v1/endpoints")
         self.session = requests.Session()
 
-        # Use the same CA as Curl to do TLS verification;
-        # if it's not defined then disable TLS verification.
-        self.session.verify = os.environ.get("CURL_CA_BUNDLE", False)
         if headers:
             self.session.headers.update(headers)
         response = self.session.get(url)

diff --git a/lib/pbench/client/oidc_admin.py b/lib/pbench/client/oidc_admin.py
@@ -12,7 +12,7 @@ class OIDCAdmin(Connection):
     ADMIN_PASSWORD = os.getenv("OIDC_ADMIN_PASSWORD", "admin")
 
     def __init__(self, server_url: str):
-        super().__init__(server_url, verify=False)
+        super().__init__(server_url)
 
     def get_admin_token(self) -> dict:
         """pbench-server realm admin user login.

diff --git a/lib/pbench/server/auth/__init__.py b/lib/pbench/server/auth/__init__.py
@@ -205,6 +205,11 @@ def wait_for_oidc_server(
         """
         try:
             oidc_server = server_config.get("openid", "server_url")
+            oidc_realm = server_config.get("openid", "realm")
+            # Get a custom cert location to verify Keycloak ssl if its define
+            # in the config file. Otherwise, we default to using system-wide
+            # certificates.
+            cert = server_config.get("openid", "cert_location", fallback=True)
         except (NoOptionError, NoSectionError) as exc:
             raise OpenIDClient.NotConfigured() from exc
 
@@ -224,18 +229,20 @@ def wait_for_oidc_server(
             status_forcelist=tuple(int(x) for x in HTTPStatus if x != 200),
         )
         adapter = HTTPAdapter(max_retries=retry)
-        session.mount("http://", adapter)
         session.mount("https://", adapter)
 
         # We will also need to retry the connection if the health status is not UP.
         connected = False
         for _ in range(5):
             try:
-                response = session.get(f"{oidc_server}/health")
+                response = session.get(
+                    f"{oidc_server}/realms/{oidc_realm}/.well-known/openid-configuration",
+                    verify=cert,
+                )
                 response.raise_for_status()
             except Exception as exc:
                 raise OpenIDClient.ServerConnectionError() from exc
-            if response.json().get("status") == "UP":
+            if response.json().get("issuer") == f"{oidc_server}/realms/{oidc_realm}":
                 logger.debug("OIDC server connection verified")
                 connected = True
                 break

diff --git a/lib/pbench/test/unit/server/auth/test_auth.py b/lib/pbench/test/unit/server/auth/test_auth.py
@@ -293,29 +293,36 @@ def test_wait_for_oidc_server_fail(self, make_logger):
         with pytest.raises(OpenIDClient.NotConfigured):
             OpenIDClient.wait_for_oidc_server(config, make_logger)
         # Missing "server_url"
-        section = {}
-        config["openid"] = section
+        config["openid"] = {}
         with pytest.raises(OpenIDClient.NotConfigured):
             OpenIDClient.wait_for_oidc_server(config, make_logger)
 
-        section = {"server_url": "https://example.com"}
-        config["openid"] = section
+        config["openid"] = {
+            "server_url": "https://example.com",
+            "realm": "realm",
+            "cert_location": "/ca.crt",
+        }
+
+        # Keycloak well-known endpoint without any response
+        with pytest.raises(OpenIDClient.ServerConnectionError):
+            OpenIDClient.wait_for_oidc_server(config, make_logger)
 
-        # Keycloak health returning response but status is not UP
+        # Keycloak well-known endpoint returning response without a valid issuer
         responses.add(
             responses.GET,
-            "https://example.com/health",
-            body='{"status": "DOWN","checks": []}',
+            "https://example.com/realms/realm/.well-known/openid-configuration",
+            body="{}",
             content_type="application/json",
         )
 
         with pytest.raises(OpenIDClient.ServerConnectionTimedOut):
             OpenIDClient.wait_for_oidc_server(config, make_logger)
 
-        # Keycloak health returning network exception and no content
+        # Keycloak well-known endpoint returning network exception and no
+        # content
         responses.add(
             responses.GET,
-            "https://example.com/health",
+            "https://example.com/realms/realm/.well-known/openid-configuration",
             body=Exception("some network exception"),
         )
 
@@ -327,14 +334,18 @@ def test_wait_for_oidc_server_succ(self, make_logger):
         """Verify .wait_for_oidc_server() success mode"""
 
         config = configparser.ConfigParser()
-        section = {"server_url": "https://example.com"}
-        config["openid"] = section
 
-        # Keycloak health returning response with status UP
+        config["openid"] = {
+            "server_url": "https://example.com",
+            "realm": "realm",
+            "cert_location": "/ca.crt",
+        }
+
+        # Keycloak well-known endpoint returning response with valid issuer
         responses.add(
             responses.GET,
-            "https://example.com/health",
-            body='{"status": "UP","checks": []}',
+            "https://example.com/realms/realm/.well-known/openid-configuration",
+            body='{"issuer": "https://example.com/realms/realm"}',
             content_type="application/json",
         )
 

diff --git a/server/lib/config/pbench-server-default.cfg b/server/lib/config/pbench-server-default.cfg
@@ -100,6 +100,10 @@ realm = pbench-server
 # Client entity name requesting OIDC to authenticate a user.
 client = pbench-client
 
+# Cert location for connecting to the OIDC client
+# If you want to use a custom CA then its location path should be recorded.
+#cert_location = /path/CA
+
 [logging]
 logger_type = devlog
 logging_level = INFO

diff --git a/server/pbenchinacan/container-build.sh b/server/pbenchinacan/container-build.sh
@@ -63,6 +63,10 @@ buildah run $container rm -f /tmp/pbench-server.rpm
 buildah copy --chown root:root --chmod 0644 $container \
     ${PBINC_SERVER}/lib/config/nginx.conf /etc/nginx/nginx.conf
 
+# Add our Pbench Server CA certificate.
+buildah copy --chown root:root --chmod 0444 $container \
+    ${PBINC_INACAN}/etc/pki/tls/certs/pbench_CA.crt /etc/pki/tls/certs/pbench_CA.crt
+
 # Since we configure Nginx to log via syslog directly, remove Nginx log rotation
 # configuration as it emits unnecessary "Permission denied" errors.
 buildah run $container rm /etc/logrotate.d/nginx

diff --git a/server/pbenchinacan/deploy b/server/pbenchinacan/deploy
@@ -44,6 +44,7 @@ PB_DEPLOY_FILES=${PB_DEPLOY_FILES:-${HOME}/Deploy}
 SRV_PBENCH=${SRV_PBENCH:-/srv/pbench}
 PB_SSL_CERT_FILE=${PB_SSL_CERT_FILE:-${PB_DEPLOY_FILES}/pbench-server.crt}
 PB_SSL_KEY_FILE=${PB_SSL_KEY_FILE:-${PB_DEPLOY_FILES}/pbench-server.key}
+PB_SSL_CA_FILE=${PB_SSL_CA_FILE:-${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt}
 
 # Locations inside the container
 #

diff --git a/server/pbenchinacan/deploy-dependencies b/server/pbenchinacan/deploy-dependencies
@@ -91,10 +91,14 @@ podman run \
   --name "${PB_POD_NAME}-keycloak" \
   --pull ${PB_KEYCLOAK_IMAGE_PULL_POLICY} \
   --restart no \
+  -v ${PB_DEPLOY_FILES}/pbench-server.crt:/opt/keycloak/conf/keycloak.crt:Z \
+  -v ${PB_DEPLOY_FILES}/pbench-server.key:/opt/keycloak/conf/keycloak.key:Z \
   -e KEYCLOAK_ADMIN=admin \
   -e KEYCLOAK_ADMIN_PASSWORD=admin \
   -d \
   ${PB_KEYCLOAK_IMAGE} \
-  start-dev --health-enabled=true --http-port=8090
+  start-dev --health-enabled=true --https-port=8090 \
+  --https-certificate-file=/opt/keycloak/conf/keycloak.crt \
+  --https-certificate-key-file=/opt/keycloak/conf/keycloak.key
 
 server/pbenchinacan/load_keycloak.sh
diff --git a/server/pbenchinacan/etc/pbench-server/pbench-server.cfg b/server/pbenchinacan/etc/pbench-server/pbench-server.cfg
@@ -24,7 +24,11 @@ uri = postgresql://pbenchcontainer:pbench@localhost:5432/pbenchcontainer
 secret-key = "pbench-in-a-can secret shhh"
 
 [openid]
-server_url = http://localhost:8090
+server_url = https://localhost:8090
+
+# Override the default cert value to use for pbenchinacan Keycloak container
+# connection.
+cert_location = /etc/pki/tls/certs/pbench_CA.crt
 
 ###########################################################################
 # The rest will come from the default config file.

diff --git a/server/pbenchinacan/load_keycloak.sh b/server/pbenchinacan/load_keycloak.sh
@@ -17,7 +17,7 @@
 # "https://localhost:8443/*" unless specified otherwise by 'KEYCLOAK_REDIRECT_URI'
 # env variable.
 
-KEYCLOAK_HOST_PORT=${KEYCLOAK_HOST_PORT:-"http://localhost:8090"}
+KEYCLOAK_HOST_PORT=${KEYCLOAK_HOST_PORT:-"https://localhost:8090"}
 KEYCLOAK_REDIRECT_URI=${KEYCLOAK_REDIRECT_URI:-"https://localhost:8443/*"}
 ADMIN_USERNAME=${ADMIN_USERNAME:-"admin"}
 ADMIN_PASSWORD=${ADMIN_PASSWORD:-"admin"}
@@ -26,13 +26,19 @@ ADMIN_PASSWORD=${ADMIN_PASSWORD:-"admin"}
 REALM=${KEYCLOAK_REALM:-"pbench-server"}
 CLIENT=${KEYCLOAK_CLIENT:-"pbench-client"}
 
+TMP_DIR=${TMP_DIR:-${WORKSPACE_TMP:-/var/tmp/pbench}}
+PB_DEPLOY_FILES=${PB_DEPLOY_FILES:-${TMP_DIR}/pbench_server_deployment}
+
+export CURL_CA_BUNDLE=${CURL_CA_BUNDLE:-"${PWD}/server/pbenchinacan/etc/pki/tls/certs/pbench_CA.crt"}
+
 end_in_epoch_secs=$(date --date "2 minutes" +%s)
 
 # Run the custom configuration
 
 ADMIN_TOKEN=""
 while true; do
-  ADMIN_TOKEN=$(curl -s -f -X POST "${KEYCLOAK_HOST_PORT}/realms/master/protocol/openid-connect/token" \
+  ADMIN_TOKEN=$(curl -s -f -X POST \
+    "${KEYCLOAK_HOST_PORT}/realms/master/protocol/openid-connect/token" \
     -H "Content-Type: application/x-www-form-urlencoded" \
     -d "username=${ADMIN_USERNAME}" \
     -d "password=${ADMIN_PASSWORD}" \
@@ -53,7 +59,8 @@ echo
 echo "Keycloak connection successful on : ${KEYCLOAK_HOST_PORT}"
 echo
 
-status_code=$(curl -f -s -o /dev/null -w "%{http_code}" -X POST "${KEYCLOAK_HOST_PORT}/admin/realms" \
+status_code=$(curl -f -s -o /dev/null -w "%{http_code}" -X POST \
+  "${KEYCLOAK_HOST_PORT}/admin/realms" \
   -H "Authorization: Bearer ${ADMIN_TOKEN}" \
   -H "Content-Type: application/json" \
   -d '{"realm": "'${REALM}'", "enabled": true}')
@@ -70,7 +77,8 @@ fi
 # a token from Keycloak using a <client_id>.
 # Having <client_id> in the aud claim of the token is essential for the token
 # to be validated.
-curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/client-scopes" \
+curl -si -f -X POST \
+  "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/client-scopes" \
   -H "Authorization: Bearer ${ADMIN_TOKEN}" \
   -H "Content-Type: application/json" \
   -d '{
@@ -99,7 +107,8 @@ curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/client-scopes"
     }'
 
 
-CLIENT_CONF=$(curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients" \
+CLIENT_CONF=$(curl -si -f -X POST \
+  "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients" \
   -H "Authorization: Bearer ${ADMIN_TOKEN}" \
   -H "Content-Type: application/json" \
   -d '{"clientId": "'${CLIENT}'",
@@ -111,15 +120,16 @@ CLIENT_CONF=$(curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/c
        "attributes": {"post.logout.redirect.uris": "'${KEYCLOAK_REDIRECT_URI}'"},
        "redirectUris": ["'${KEYCLOAK_REDIRECT_URI}'"]}')
 
-CLIENT_ID=$(grep -o -e 'http://[^[:space:]]*' <<< ${CLIENT_CONF} | sed -e 's|.*/||')
+CLIENT_ID=$(grep -o -e 'https://[^[:space:]]*' <<< ${CLIENT_CONF} | sed -e 's|.*/||')
 if [[ -z "${CLIENT_ID}" ]]; then
   echo "${CLIENT} id is empty"
   exit 1
 else
   echo "Created ${CLIENT} client"
 fi
 
-status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients/${CLIENT_ID}/roles" \
+status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \
+  "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/clients/${CLIENT_ID}/roles" \
   -H "Authorization: Bearer ${ADMIN_TOKEN}" \
   -H "Content-Type: application/json" \
   -d '{"name": "ADMIN"}')
@@ -139,12 +149,13 @@ if [[ -z "${ROLE_ID}" ]]; then
   exit 1
 fi
 
-USER=$(curl -si -f -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users" \
+USER=$(curl -si -f -X POST \
+  "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users" \
   -H "Authorization: Bearer ${ADMIN_TOKEN}" \
   -H "Content-Type: application/json" \
   -d '{"username": "admin", "enabled": true, "credentials": [{"type": "password", "value": "123", "temporary": false}]}')
 
-USER_ID=$(grep -o -e 'http://[^[:space:]]*' <<< ${USER} | sed -e 's|.*/||')
+USER_ID=$(grep -o -e 'https://[^[:space:]]*' <<< ${USER} | sed -e 's|.*/||')
 
 if [[ -z "${USER_ID}" ]]; then
   echo "User id is empty"
@@ -153,7 +164,8 @@ else
   echo "Created an 'admin' user inside ${REALM} realm"
 fi
 
-status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users/${USER_ID}/role-mappings/clients/${CLIENT_ID}" \
+status_code=$(curl -s -o /dev/null -w "%{http_code}" -X POST \
+  "${KEYCLOAK_HOST_PORT}/admin/realms/${REALM}/users/${USER_ID}/role-mappings/clients/${CLIENT_ID}" \
   -H "Authorization: Bearer ${ADMIN_TOKEN}" \
   -H "Content-Type: application/json" \
   -d '[{"id":"'${ROLE_ID}'","name":"ADMIN"}]')

diff --git a/server/pbenchinacan/run-pbench-in-a-can b/server/pbenchinacan/run-pbench-in-a-can
@@ -131,9 +131,10 @@ podman run \
     -addext "authorityKeyIdentifier = keyid,issuer" \
     -addext "basicConstraints=CA:FALSE" \
     -addext "keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment" \
-    -addext "subjectAltName = IP.2:${PB_HOST_IP}" \
+    -addext "subjectAltName = IP.2:${PB_HOST_IP}, DNS:localhost" \
     2>&1 | sed -E -e '/^[.+*-]*$/ d'
 
+chmod 0640 ${PB_DEPLOY_FILES}/pbench-server.key
 #+
 # Start the services which the Pbench Server depends upon and then start the
 # Pbench Server itself.

diff --git a/tox.ini b/tox.ini
@@ -21,7 +21,6 @@ passenv =
     USER
     WORKSPACE
     WORKSPACE_TMP
-
 setenv =
     VIRTUAL_ENV = {envdir}
     XDG_CACHE_HOME = {envdir}