Skip to content

Commit

Permalink
[#3537] Get K8s container id via K8s API.
Browse files Browse the repository at this point in the history
Signed-off-by: Carsten Lohmann <carsten.lohmann@bosch.io>
  • Loading branch information
calohmn committed Sep 22, 2023
1 parent 817db3a commit 5274852
Show file tree
Hide file tree
Showing 4 changed files with 341 additions and 1 deletion.
15 changes: 15 additions & 0 deletions clients/command/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,28 @@
<artifactId>opentracing-noop</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-kubernetes-client</artifactId>
<exclusions>
<exclusion>
<groupId>org.jboss.spec.javax.xml.bind</groupId>
<artifactId>jboss-jaxb-api_2.3_spec</artifactId>
</exclusion>
</exclusions>
</dependency>

<!-- testing -->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-server-mock</artifactId>
<scope>test</scope>
</dependency>

</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ private CommandRoutingUtil() {
* @return The new adapter instance identifier.
*/
public static String getNewAdapterInstanceId(final String adapterName, final int counter) {
final String k8sContainerId = CgroupV1KubernetesContainerUtil.getContainerId();
final String k8sContainerId = KubernetesContainerUtil.getContainerId();
if (k8sContainerId == null || k8sContainerId.length() < 12) {
return getNewAdapterInstanceIdForNonK8sEnv(adapterName);
} else {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,184 @@
/*******************************************************************************
* Copyright (c) 2023 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*******************************************************************************/

package org.eclipse.hono.client.command;

import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import io.fabric8.kubernetes.api.model.ContainerStatus;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.KubernetesClientBuilder;
import io.fabric8.kubernetes.client.KubernetesClientException;
import io.fabric8.kubernetes.client.utils.PodStatusUtil;

/**
* A helper class for detecting the container ID if running in a Kubernetes container.
*/
public final class KubernetesContainerUtil {

/**
* Name of the environment variable that contains the name of the container that this application is running in.
* Such an environment variable needs to be set if the pod that this application is running in contains multiple
* running containers.
*/
public static final String KUBERNETES_CONTAINER_NAME_ENV_VAR = "KUBERNETES_CONTAINER_NAME";

private static final Logger LOG = LoggerFactory.getLogger(KubernetesContainerUtil.class);

private static final int NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS = 3;

private static String containerId;

private KubernetesContainerUtil() {
}

/**
* Returns the container id if running in a container in Kubernetes.
* <p>
* First an attempt is made to get the container id by inspecting <code>/proc/self/cgroup</code>
* (via {@link CgroupV1KubernetesContainerUtil#getContainerId()}).
* If not found there, the container id is queried via the Kubernetes API.
* <p>
* NOTE: The service account of the application pod must have an RBAC role allowing "get" on the "pods" resource.
* If this application is running in a pod with multiple containers, the container that this application is running
* in must have an environment variable with the name specified in {@link #KUBERNETES_CONTAINER_NAME_ENV_VAR} set
* to the container name.
*
* @return The container id or {@code null} if not running in Kubernetes.
* @throws RuntimeException If getting the container id via the K8s API failed.
* @throws IllegalStateException If there was an error getting the container id via the K8s API because of missing
* permissions or because multiple pod containers exist and no KUBERNETES_CONTAINER_NAME env var is set.
*/
public static String getContainerId() throws RuntimeException {
if (containerId != null) {
return containerId;
}
if (!runningInKubernetes()) {
return null;
}
final String containerIdViaCgroup1 = CgroupV1KubernetesContainerUtil.getContainerId();
if (containerIdViaCgroup1 != null) {
containerId = containerIdViaCgroup1;
return containerId;
}
int attempt = 0;
while (++attempt <= NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS && containerId == null) {
try {
containerId = getContainerIdViaK8sApi();
} catch (final KubernetesClientException e) {
LOG.error("[attempt {} of {}] Error getting container id via K8s API", attempt,
NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS, e);
if (attempt == NUM_CONTAINER_ID_RETRIEVAL_ATTEMPTS) {
if (e.getCause() != null && e.getCause().getMessage() != null
&& e.getCause().getMessage().contains("timed out")) {
throw new RuntimeException("Timed out getting container id via K8s API. " +
"Consider increasing the request timeout via the KUBERNETES_REQUEST_TIMEOUT env var (default is 10000[ms]).");
}
throw new RuntimeException("Error getting container id via K8s API: " + e.getMessage());
}
}
}
return containerId;
}

private static boolean runningInKubernetes() {
return System.getenv("KUBERNETES_SERVICE_HOST") != null;
}

private static String getContainerIdViaK8sApi() throws KubernetesClientException {
LOG.info("get container id via K8s API");
try (KubernetesClient client = new KubernetesClientBuilder().build()) {
final String podName = System.getenv("HOSTNAME");
// container name env var needs to be set if there are multiple running containers in the pod
final String containerNameEnvVarValue = System.getenv(KUBERNETES_CONTAINER_NAME_ENV_VAR);
return getContainerIdViaK8sApi(client, podName, containerNameEnvVarValue);
}
}

static String getContainerIdViaK8sApi(final KubernetesClient client, final String podName,
final String containerNameEnvVarValue) throws KubernetesClientException {
// Note: any exceptions that should trigger a retry are thrown as KubernetesClientException here
try {
final String namespace = Optional.ofNullable(client.getNamespace()).orElse("default");
final Pod pod = client.pods().inNamespace(namespace).withName(podName).get();
if (pod == null) {
throw new KubernetesClientException("application pod not found in Kubernetes namespace " + namespace);
}
final List<ContainerStatus> containerStatuses = PodStatusUtil.getContainerStatus(pod).stream()
.filter(KubernetesContainerUtil::isContainerRunning).toList();
if (containerStatuses.isEmpty()) {
LOG.info("got container status list {}", containerStatuses);
throw new KubernetesClientException(
"no running container found in pod %s, namespace %s".formatted(podName, namespace));
}
final ContainerStatus foundContainerStatus;
if (containerStatuses.size() > 1) {
final String foundContainerNames = containerStatuses.stream().map(ContainerStatus::getName)
.collect(Collectors.joining(", "));
if (containerNameEnvVarValue == null) {
LOG.error(
"can't get container id: found multiple running containers, but {} env var is not set " +
"to specify which container to use; found containers [{}] in pod {}",
KUBERNETES_CONTAINER_NAME_ENV_VAR, foundContainerNames, podName);
throw new IllegalStateException(
("can't get container id via K8s API: multiple running containers found; " +
"the %s env variable needs to be set for the container this application is running in, " +
"having the container name as value")
.formatted(KUBERNETES_CONTAINER_NAME_ENV_VAR));
}
LOG.info("multiple running containers found: {}", foundContainerNames);
LOG.info("using container name {} (derived from env var {}) to determine container id",
containerNameEnvVarValue, KUBERNETES_CONTAINER_NAME_ENV_VAR);
foundContainerStatus = containerStatuses.stream()
.filter(status -> status.getName().equals(containerNameEnvVarValue))
.findFirst()
.orElseThrow(() -> new KubernetesClientException(
"no running container with name %s found in pod %s, namespace %s"
.formatted(containerNameEnvVarValue, podName, namespace)));
} else {
foundContainerStatus = containerStatuses.get(0);
}
String containerId = foundContainerStatus.getContainerID();
// remove container runtime prefix (e.g. "containerd://")
final int delimIdx = containerId.lastIndexOf("://");
if (delimIdx > -1) {
containerId = containerId.substring(delimIdx + 3);
}
LOG.info("got container id via K8s API: {}", containerId);
return containerId;

} catch (final KubernetesClientException e) {
// rethrow error concerning missing RBAC role assignment as IllegalStateException to skip retry
// Error message looks like this:
// Forbidden!Configured service account doesn't have access. Service account may have been revoked. pods "XXX" is forbidden:
// User "XXX" cannot get resource "pods" in API group "" in the namespace "hono".
if (e.getMessage().contains("orbidden")) {
LOG.error("Error getting container id via K8s API: \n{}", e.getMessage());
throw new IllegalStateException("error getting container id via K8s API: " +
"application pod needs service account with role binding allowing 'get' on 'pods' resource");
}
throw e;
}
}

private static boolean isContainerRunning(final ContainerStatus containerStatus) {
return containerStatus.getState() != null
&& containerStatus.getState().getRunning() != null;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
/*******************************************************************************
* Copyright (c) 2023 Contributors to the Eclipse Foundation
*
* See the NOTICE file(s) distributed with this work for additional
* information regarding copyright ownership.
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Public License 2.0 which is available at
* http://www.eclipse.org/legal/epl-2.0
*
* SPDX-License-Identifier: EPL-2.0
*******************************************************************************/

package org.eclipse.hono.client.command;

import static org.junit.jupiter.api.Assertions.assertThrows;

import static com.google.common.truth.Truth.assertThat;

import java.util.List;
import java.util.UUID;

import org.junit.jupiter.api.Test;

import io.fabric8.kubernetes.api.model.ContainerStateBuilder;
import io.fabric8.kubernetes.api.model.ContainerStatus;
import io.fabric8.kubernetes.api.model.ContainerStatusBuilder;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.PodBuilder;
import io.fabric8.kubernetes.api.model.PodStatus;
import io.fabric8.kubernetes.api.model.PodStatusBuilder;
import io.fabric8.kubernetes.client.KubernetesClient;
import io.fabric8.kubernetes.client.server.mock.EnableKubernetesMockClient;
import io.fabric8.kubernetes.client.server.mock.KubernetesMockServer;

/**
* Unit tests for {@link KubernetesContainerUtil}.
*/
@EnableKubernetesMockClient(https = false)
public class KubernetesContainerUtilTest {

KubernetesMockServer server;
/**
* This client uses the namespace "test" (see KubernetesMockServer#createClient()).
*/
KubernetesClient client;

/**
* Tests getting the container id via the K8s API.
*/
@Test
public void testGetContainerIdViaK8sApi() {
final String podName = "testPod0";
final String containerId = getRandomContainerId();
final String containerIdWithPrefix = "containerd://" + containerId;
final String containerNameEnvVarValue = null; // should not be needed in this test (only one running container)

final ContainerStatus containerStatus = createRunningContainerStatus(containerIdWithPrefix, "testContainer0");
final Pod pod = createPod(podName, List.of(containerStatus));
server.expect()
.withPath("/api/v1/namespaces/test/pods/" + podName)
.andReturn(200, pod)
.once();
final String extractedContainerId = KubernetesContainerUtil.getContainerIdViaK8sApi(client, podName, containerNameEnvVarValue);
assertThat(extractedContainerId).isEqualTo(containerId);
}

/**
* Tests getting the container id via the K8s API with multiple running pods.
*/
@Test
public void testGetContainerIdViaK8sApiWithMultipleRunningContainers() {
final String podName = "testPod0";
final String containerId1WithPrefix = "containerd://" + getRandomContainerId();
final String containerId2 = getRandomContainerId();
final String containerId2WithPrefix = "containerd://" + containerId2;
final String containerName1 = "testContainer1";
final String containerName2 = "testContainer2";
final ContainerStatus containerStatus1 = createRunningContainerStatus(containerId1WithPrefix, containerName1);
final ContainerStatus containerStatus2 = createRunningContainerStatus(containerId2WithPrefix, containerName2);

final Pod pod = createPod(podName, List.of(containerStatus1, containerStatus2));
server.expect()
.withPath("/api/v1/namespaces/test/pods/" + podName)
.andReturn(200, pod)
.once();
final String extractedContainerId = KubernetesContainerUtil.getContainerIdViaK8sApi(client, podName, containerName2);
assertThat(extractedContainerId).isEqualTo(containerId2);
}

/**
* Tests getting the container id via the K8s API with multiple running pods, but no environment variable set
* to specify the container name.
*/
@Test
public void testGetContainerIdViaK8sApiWithMultipleRunningContainersButNoContainerNameEnvVar() {
final String podName = "testPod0";
final String containerId1 = getRandomContainerId();
final String containerId2 = getRandomContainerId();
final String containerName1 = "testContainer1";
final String containerName2 = "testContainer2";
final ContainerStatus containerStatus1 = createRunningContainerStatus(containerId1, containerName1);
final ContainerStatus containerStatus2 = createRunningContainerStatus(containerId2, containerName2);
final String containerNameEnvVarValue = null;

final Pod pod = createPod(podName, List.of(containerStatus1, containerStatus2));
server.expect()
.withPath("/api/v1/namespaces/test/pods/" + podName)
.andReturn(200, pod)
.once();
final IllegalStateException thrown = assertThrows(
IllegalStateException.class,
() -> KubernetesContainerUtil.getContainerIdViaK8sApi(client, podName, containerNameEnvVarValue));
assertThat(thrown.getMessage()).contains("multiple running containers");
}

private static ContainerStatus createRunningContainerStatus(final String containerIdWithPrefix,
final String containerName) {
return new ContainerStatusBuilder()
.withContainerID(containerIdWithPrefix)
.withName(containerName)
.withState(new ContainerStateBuilder().withNewRunning().endRunning().build())
.build();
}

private Pod createPod(final String podName, final List<ContainerStatus> containerStatuses) {
final PodStatus podStatus = new PodStatusBuilder()
.withContainerStatuses(containerStatuses)
.build();
return new PodBuilder()
.withNewMetadata()
.withName(podName)
.endMetadata()
.withStatus(podStatus)
.build();
}

private static String getRandomContainerId() {
return UUID.randomUUID().toString().concat(UUID.randomUUID().toString()).replaceAll("-", "");
}
}

0 comments on commit 5274852

Please sign in to comment.