[#3537] Get K8s container id via K8s API.

Signed-off-by: Carsten Lohmann <carsten.lohmann@bosch.io>

clients/command/pom.xml

@@ -67,13 +67,28 @@
       <artifactId>opentracing-noop</artifactId>
       <optional>true</optional>
     </dependency>
+    <dependency>
+      <groupId>io.quarkus</groupId>
+      <artifactId>quarkus-kubernetes-client</artifactId>
+      <exclusions>
+        <exclusion>
+          <groupId>org.jboss.spec.javax.xml.bind</groupId>
+          <artifactId>jboss-jaxb-api_2.3_spec</artifactId>
+        </exclusion>
+      </exclusions>
+    </dependency>
 
     <!-- testing -->
     <dependency>
       <groupId>ch.qos.logback</groupId>
       <artifactId>logback-classic</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>io.fabric8</groupId>
+      <artifactId>kubernetes-server-mock</artifactId>
+      <scope>test</scope>
+    </dependency>
 
   </dependencies>
 </project>

clients/command/src/main/java/org/eclipse/hono/client/command/CommandRoutingUtil.java

@@ -54,7 +54,7 @@ private CommandRoutingUtil() {
      * @return The new adapter instance identifier.
      */
     public static String getNewAdapterInstanceId(final String adapterName, final int counter) {
-        final String k8sContainerId = CgroupV1KubernetesContainerUtil.getContainerId();
+        final String k8sContainerId = KubernetesContainerUtil.getContainerId();
         if (k8sContainerId == null || k8sContainerId.length() < 12) {
             return getNewAdapterInstanceIdForNonK8sEnv(adapterName);
         } else {

clients/command/src/main/java/org/eclipse/hono/client/command/KubernetesContainerUtil.java

@@ -0,0 +1,184 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+
+package org.eclipse.hono.client.command;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import io.fabric8.kubernetes.api.model.ContainerStatus;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.KubernetesClientBuilder;
+import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.utils.PodStatusUtil;
+
+/**
+ * A helper class for detecting the container ID if running in a Kubernetes container.
+ */
+public final class KubernetesContainerUtil {
+
+    /**
+     * Name of the environment variable that contains the name of the container that this application is running in.
+     * Such an environment variable needs to be set if the pod that this application is running in contains multiple
+     * running containers.
+     */
+    public static final String KUBERNETES_CONTAINER_NAME_ENV_VAR = "KUBERNETES_CONTAINER_NAME";
+
+    private static final Logger LOG = LoggerFactory.getLogger(KubernetesContainerUtil.class);
+
+    private static final int NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS = 3;
+
+    private static String containerId;
+
+    private KubernetesContainerUtil() {
+    }
+
+    /**
+     * Returns the container id if running in a container in Kubernetes.
+     * <p>
+     * First an attempt is made to get the container id by inspecting <code>/proc/self/cgroup</code>
+     * (via {@link CgroupV1KubernetesContainerUtil#getContainerId()}).
+     * If not found there, the container id is queried via the Kubernetes API.
+     * <p>
+     * NOTE: The service account of the application pod must have an RBAC role allowing "get" on the "pods" resource.
+     * If this application is running in a pod with multiple containers, the container that this application is running
+     * in must have an environment variable with the name specified in {@link #KUBERNETES_CONTAINER_NAME_ENV_VAR} set
+     * to the container name.
+     *
+     * @return The container id or {@code null} if not running in Kubernetes.
+     * @throws RuntimeException If getting the container id via the K8s API failed.
+     * @throws IllegalStateException If there was an error getting the container id via the K8s API because of missing
+     *             permissions or because multiple pod containers exist and no KUBERNETES_CONTAINER_NAME env var is set.
+     */
+    public static String getContainerId() throws RuntimeException {
+        if (containerId != null) {
+            return containerId;
+        }
+        if (!runningInKubernetes()) {
+            return null;
+        }
+        final String containerIdViaCgroup1 = CgroupV1KubernetesContainerUtil.getContainerId();
+        if (containerIdViaCgroup1 != null) {
+            containerId = containerIdViaCgroup1;
+            return containerId;
+        }
+        int attempt = 0;
+        while (++attempt <= NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS && containerId == null) {
+            try {
+                containerId = getContainerIdViaK8sApi();
+            } catch (final KubernetesClientException e) {
+                LOG.error("[attempt {} of {}] Error getting container id via K8s API", attempt,
+                        NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS, e);
+                if (attempt == NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS) {
+                    if (e.getCause() != null && e.getCause().getMessage() != null
+                            && e.getCause().getMessage().contains("timed out")) {
+                        throw new RuntimeException("Timed out getting container id via K8s API. " +
+                                "Consider increasing the request timeout via the KUBERNETES_REQUEST_TIMEOUT env var (default is 10000[ms]).");
+                    }
+                    throw new RuntimeException("Error getting container id via K8s API: " + e.getMessage());
+                }
+            }
+        }
+        return containerId;
+    }
+
+    private static boolean runningInKubernetes() {
+        return System.getenv("KUBERNETES_SERVICE_HOST") != null;
+    }
+
+    private static String getContainerIdViaK8sApi() throws KubernetesClientException {
+        LOG.info("get container id via K8s API");
+        try (KubernetesClient client = new KubernetesClientBuilder().build()) {
+            final String podName = System.getenv("HOSTNAME");
+            // container name env var needs to be set if there are multiple running containers in the pod
+            final String containerNameEnvVarValue = System.getenv(KUBERNETES_CONTAINER_NAME_ENV_VAR);
+            return getContainerIdViaK8sApi(client, podName, containerNameEnvVarValue);
+        }
+    }
+
+    static String getContainerIdViaK8sApi(final KubernetesClient client, final String podName,
+            final String containerNameEnvVarValue) throws KubernetesClientException {
+        // Note: any exceptions that should trigger a retry are thrown as KubernetesClientException here
+        try {
+            final String namespace = Optional.ofNullable(client.getNamespace()).orElse("default");
+            final Pod pod = client.pods().inNamespace(namespace).withName(podName).get();
+            if (pod == null) {
+                throw new KubernetesClientException("application pod not found in Kubernetes namespace " + namespace);
+            }
+            final List<ContainerStatus> containerStatuses = PodStatusUtil.getContainerStatus(pod).stream()
+                    .filter(KubernetesContainerUtil::isContainerRunning).toList();
+            if (containerStatuses.isEmpty()) {
+                LOG.info("got container status list {}", containerStatuses);
+                throw new KubernetesClientException(
+                        "no running container found in pod %s, namespace %s".formatted(podName, namespace));
+            }
+            final ContainerStatus foundContainerStatus;
+            if (containerStatuses.size() > 1) {
+                final String foundContainerNames = containerStatuses.stream().map(ContainerStatus::getName)
+                        .collect(Collectors.joining(", "));
+                if (containerNameEnvVarValue == null) {
+                    LOG.error(
+                            "can't get container id: found multiple running containers, but {} env var is not set " +
+                                    "to specify which container to use; found containers [{}] in pod {}",
+                            KUBERNETES_CONTAINER_NAME_ENV_VAR, foundContainerNames, podName);
+                    throw new IllegalStateException(
+                            ("can't get container id via K8s API: multiple running containers found; " +
+                                    "the %s env variable needs to be set for the container this application is running in, " +
+                                    "having the container name as value")
+                                    .formatted(KUBERNETES_CONTAINER_NAME_ENV_VAR));
+                }
+                LOG.info("multiple running containers found: {}", foundContainerNames);
+                LOG.info("using container name {} (derived from env var {}) to determine container id",
+                        containerNameEnvVarValue, KUBERNETES_CONTAINER_NAME_ENV_VAR);
+                foundContainerStatus = containerStatuses.stream()
+                        .filter(status -> status.getName().equals(containerNameEnvVarValue))
+                        .findFirst()
+                        .orElseThrow(() -> new KubernetesClientException(
+                                "no running container with name %s found in pod %s, namespace %s"
+                                        .formatted(containerNameEnvVarValue, podName, namespace)));
+            } else {
+                foundContainerStatus = containerStatuses.get(0);
+            }
+            String containerId = foundContainerStatus.getContainerID();
+            // remove container runtime prefix (e.g. "containerd://")
+            final int delimIdx = containerId.lastIndexOf("://");
+            if (delimIdx > -1) {
+                containerId = containerId.substring(delimIdx + 3);
+            }
+            LOG.info("got container id via K8s API: {}", containerId);
+            return containerId;
+
+        } catch (final KubernetesClientException e) {
+            // rethrow error concerning missing RBAC role assignment as IllegalStateException to skip retry
+            // Error message looks like this:
+            // Forbidden!Configured service account doesn't have access. Service account may have been revoked. pods "XXX" is forbidden:
+            // User "XXX" cannot get resource "pods" in API group "" in the namespace "hono".
+            if (e.getMessage().contains("orbidden")) {
+                LOG.error("Error getting container id via K8s API: \n{}", e.getMessage());
+                throw new IllegalStateException("error getting container id via K8s API: " +
+                        "application pod needs service account with role binding allowing 'get' on 'pods' resource");
+            }
+            throw e;
+        }
+    }
+
+    private static boolean isContainerRunning(final ContainerStatus containerStatus) {
+        return containerStatus.getState() != null
+                && containerStatus.getState().getRunning() != null;
+    }
+}

clients/command/src/test/java/org/eclipse/hono/client/command/KubernetesContainerUtilTest.java

@@ -0,0 +1,141 @@
+/*******************************************************************************
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *******************************************************************************/
+
+package org.eclipse.hono.client.command;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import java.util.List;
+import java.util.UUID;
+
+import org.junit.jupiter.api.Test;
+
+import io.fabric8.kubernetes.api.model.ContainerStateBuilder;
+import io.fabric8.kubernetes.api.model.ContainerStatus;
+import io.fabric8.kubernetes.api.model.ContainerStatusBuilder;
+import io.fabric8.kubernetes.api.model.Pod;
+import io.fabric8.kubernetes.api.model.PodBuilder;
+import io.fabric8.kubernetes.api.model.PodStatus;
+import io.fabric8.kubernetes.api.model.PodStatusBuilder;
+import io.fabric8.kubernetes.client.KubernetesClient;
+import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
+import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer;
+
+/**
+ * Unit tests for {@link KubernetesContainerUtil}.
+ */
+@EnableKubernetesMockClient(https = false)
+public class KubernetesContainerUtilTest {
+
+    KubernetesMockServer server;
+    /**
+     * This client uses the namespace "test" (see KubernetesMockServer#createClient()).
+     */
+    KubernetesClient client;
+
+    /**
+     * Tests getting the container id via the K8s API.
+     */
+    @Test
+    public void testGetContainerIdViaK8sApi() {
+        final String podName = "testPod0";
+        final String containerId = getRandomContainerId();
+        final String containerIdWithPrefix = "containerd://" + containerId;
+        final String containerNameEnvVarValue = null; // should not be needed in this test (only one running container)
+
+        final ContainerStatus containerStatus = createRunningContainerStatus(containerIdWithPrefix, "testContainer0");
+        final Pod pod = createPod(podName, List.of(containerStatus));
+        server.expect()
+                .withPath("/api/v1/namespaces/test/pods/" + podName)
+                .andReturn(200, pod)
+                .once();
+        final String extractedContainerId = KubernetesContainerUtil.getContainerIdViaK8sApi(client, podName, containerNameEnvVarValue);
+        assertThat(extractedContainerId).isEqualTo(containerId);
+    }
+
+    /**
+     * Tests getting the container id via the K8s API with multiple running pods.
+     */
+    @Test
+    public void testGetContainerIdViaK8sApiWithMultipleRunningContainers() {
+        final String podName = "testPod0";
+        final String containerId1WithPrefix = "containerd://" + getRandomContainerId();
+        final String containerId2 = getRandomContainerId();
+        final String containerId2WithPrefix = "containerd://" + containerId2;
+        final String containerName1 = "testContainer1";
+        final String containerName2 = "testContainer2";
+        final ContainerStatus containerStatus1 = createRunningContainerStatus(containerId1WithPrefix, containerName1);
+        final ContainerStatus containerStatus2 = createRunningContainerStatus(containerId2WithPrefix, containerName2);
+
+        final Pod pod = createPod(podName, List.of(containerStatus1, containerStatus2));
+        server.expect()
+                .withPath("/api/v1/namespaces/test/pods/" + podName)
+                .andReturn(200, pod)
+                .once();
+        final String extractedContainerId = KubernetesContainerUtil.getContainerIdViaK8sApi(client, podName, containerName2);
+        assertThat(extractedContainerId).isEqualTo(containerId2);
+    }
+
+    /**
+     * Tests getting the container id via the K8s API with multiple running pods, but no environment variable set
+     * to specify the container name.
+     */
+    @Test
+    public void testGetContainerIdViaK8sApiWithMultipleRunningContainersButNoContainerNameEnvVar() {
+        final String podName = "testPod0";
+        final String containerId1 = getRandomContainerId();
+        final String containerId2 = getRandomContainerId();
+        final String containerName1 = "testContainer1";
+        final String containerName2 = "testContainer2";
+        final ContainerStatus containerStatus1 = createRunningContainerStatus(containerId1, containerName1);
+        final ContainerStatus containerStatus2 = createRunningContainerStatus(containerId2, containerName2);
+        final String containerNameEnvVarValue = null;
+
+        final Pod pod = createPod(podName, List.of(containerStatus1, containerStatus2));
+        server.expect()
+                .withPath("/api/v1/namespaces/test/pods/" + podName)
+                .andReturn(200, pod)
+                .once();
+        final IllegalStateException thrown = assertThrows(
+                IllegalStateException.class,
+                () -> KubernetesContainerUtil.getContainerIdViaK8sApi(client, podName, containerNameEnvVarValue));
+        assertThat(thrown.getMessage()).contains("multiple running containers");
+    }
+
+    private static ContainerStatus createRunningContainerStatus(final String containerIdWithPrefix,
+            final String containerName) {
+        return new ContainerStatusBuilder()
+                .withContainerID(containerIdWithPrefix)
+                .withName(containerName)
+                .withState(new ContainerStateBuilder().withNewRunning().endRunning().build())
+                .build();
+    }
+
+    private Pod createPod(final String podName, final List<ContainerStatus> containerStatuses) {
+        final PodStatus podStatus = new PodStatusBuilder()
+                .withContainerStatuses(containerStatuses)
+                .build();
+        return new PodBuilder()
+                .withNewMetadata()
+                .withName(podName)
+                .endMetadata()
+                .withStatus(podStatus)
+                .build();
+    }
+
+    private static String getRandomContainerId() {
+        return UUID.randomUUID().toString().concat(UUID.randomUUID().toString()).replaceAll("-", "");
+    }
+}