fix(kube/test): Use kind instead of k3s (backport #5421) (#5451)

* fix(kube/test): Use kind instead of k3s

* fix(kube/test): Added README

* fix(kube/test): Enabled cache

* fix(kube/test): Delete test cluster after all tests finish

Co-authored-by: German Muzquiz <35276119+german-muzquiz@users.noreply.github.com>

clouddriver-kubernetes/clouddriver-kubernetes.gradle

@@ -117,8 +117,7 @@ task integrationTest(type: Test) {
   description = 'Runs kubernetes provider integration tests.'
   group = 'verification'
 
-  environment "KUBECTL_PATH", "$project.rootDir/clouddriver-kubernetes/src/integration/resources/kubectl-wrapper.sh"
-  environment "KUBECONFIGS_HOME", "$project.rootDir/clouddriver-kubernetes/build/kubeconfigs"
+  environment "IT_BUILD_HOME", "$project.buildDir/it"
   useJUnitPlatform()
 
   testClassesDirs = sourceSets.integration.output.classesDirs

clouddriver-kubernetes/src/integration/README.md

@@ -0,0 +1,18 @@
+### Kubernetes provider integration tests
+
+#### Run
+
+From the command line
+```shell
+./gradlew :clouddriver-kubernetes:integrationTest
+```
+
+From Intellij: Individual tests can be run or debugged by clicking the corresponding icon next to the test name within the IDE.
+
+
+#### How they work
+
+The tests use spring test framework to start clouddriver on a random port, reading configuration from the `clouddriver.yml` config file in the resources folder. They use testcontainers framework for starting a real mysql server in a docker container, and use [kind](https://kind.sigs.k8s.io) for starting a real kubernetes cluster where deployments will happen.
+
+Kind and kubectl binaries are downloaded to `clouddriver-kubernetes/build/it` folder, and also the `kubeconfig` file for connecting to the test cluster is generated there, which runs as a docker container started by kind.
+

clouddriver-kubernetes/src/integration/java/com/netflix/spinnaker/clouddriver/kubernetes/it/BaseTest.java

@@ -21,9 +21,11 @@
 
 import com.netflix.spinnaker.clouddriver.Main;
 import com.netflix.spinnaker.clouddriver.kubernetes.it.containers.KubernetesCluster;
+import com.netflix.spinnaker.clouddriver.kubernetes.it.utils.TestLifecycleListener;
 import io.restassured.RestAssured;
 import io.restassured.response.Response;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.extension.ExtendWith;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.boot.web.server.LocalServerPort;
 import org.springframework.test.context.TestPropertySource;
@@ -32,6 +34,7 @@
     classes = {Main.class},
     webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
 @TestPropertySource(properties = {"spring.config.location = classpath:clouddriver.yml"})
+@ExtendWith(TestLifecycleListener.class)
 public abstract class BaseTest {
 
   public static final String APP1_NAME = "testApp1";

clouddriver-kubernetes/src/integration/java/com/netflix/spinnaker/clouddriver/kubernetes/it/DeployManifestIT.java

@@ -24,12 +24,13 @@
 import com.netflix.spinnaker.clouddriver.kubernetes.it.utils.KubeTestUtils;
 import io.restassured.response.Response;
 import java.io.IOException;
-import java.util.*;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
 import org.apache.logging.log4j.util.Strings;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
-import org.testcontainers.containers.Container;
 
 public class DeployManifestIT extends BaseTest {
 
@@ -957,20 +958,6 @@ public void shouldDeployRedBlackMultidoc() throws IOException, InterruptedExcept
             .asList();
     KubeTestUtils.disableManifest(baseUrl(), body, account1Ns, "replicaSet " + appName + "-v000");
 
-    // ------------------------- then --------------------------
-    String port =
-        kubeCluster.execKubectl(
-            "-n "
-                + account1Ns
-                + " get service "
-                + SERVICE_1_NAME
-                + " -o=jsonpath='{.spec.ports[0].nodePort}'");
-    Container.ExecResult result =
-        kubeCluster.execInContainer("wget", "http://localhost:" + port, "-O", "-");
-    assertEquals(
-        0,
-        result.getExitCode(),
-        "stdout: " + result.getStdout() + " stderr: " + result.getStderr());
     List<String> podNames =
         Splitter.on(" ")
             .splitToList(
@@ -1045,19 +1032,6 @@ public void shouldDeployRedBlackReplicaSet() throws IOException, InterruptedExce
     KubeTestUtils.disableManifest(baseUrl(), body, account1Ns, "replicaSet " + appName + "-v000");
 
     // ------------------------- then --------------------------
-    String port =
-        kubeCluster.execKubectl(
-            "-n "
-                + account1Ns
-                + " get service "
-                + SERVICE_1_NAME
-                + " -o=jsonpath='{.spec.ports[0].nodePort}'");
-    Container.ExecResult result =
-        kubeCluster.execInContainer("wget", "http://localhost:" + port, "-O", " -");
-    assertEquals(
-        0,
-        result.getExitCode(),
-        "stdout: " + result.getStdout() + " stderr: " + result.getStderr());
     List<String> podNames =
         Splitter.on(" ")
             .splitToList(

clouddriver-kubernetes/src/integration/java/com/netflix/spinnaker/clouddriver/kubernetes/it/PatchManifestIT.java

@@ -21,7 +21,7 @@ public static void setUpAll() throws IOException, InterruptedException {
   }
 
   @BeforeEach
-  private void deployIfMissing() throws InterruptedException, IOException {
+  public void deployIfMissing() throws InterruptedException, IOException {
     KubeTestUtils.deployIfMissing(
         baseUrl(),
         ACCOUNT1_NAME,
@@ -58,7 +58,7 @@ public void shouldPatchManifestFromText() throws IOException, InterruptedExcepti
                 + DEPLOYMENT_1_NAME
                 + " -o=jsonpath='{.spec.template.metadata.labels}'");
     assertTrue(
-        labels.contains("testPatch:success"),
+        labels.contains("\"testPatch\":\"success\""),
         "Expected patch to add label 'testPatch' with value 'success' to "
             + DEPLOYMENT_1_NAME
             + " deployment. Labels:\n"

clouddriver-kubernetes/src/integration/java/com/netflix/spinnaker/clouddriver/kubernetes/it/containers/KubernetesCluster.java

@@ -22,24 +22,25 @@
 
 import com.google.gson.Gson;
 import java.io.*;
+import java.net.URL;
+import java.nio.channels.Channels;
+import java.nio.channels.ReadableByteChannel;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.*;
 import java.util.concurrent.TimeUnit;
 import org.springframework.util.FileCopyUtils;
-import org.testcontainers.containers.GenericContainer;
-import org.testcontainers.containers.wait.strategy.Wait;
-import org.yaml.snakeyaml.Yaml;
 
-public class KubernetesCluster extends GenericContainer<KubernetesCluster> {
-
-  private static final String DOCKER_IMAGE = "rancher/k3s:v1.17.11-k3s1";
-  private static final String KUBECFG_IN_CONTAINER = "/etc/rancher/k3s/k3s.yaml";
+public class KubernetesCluster {
 
   private static KubernetesCluster INSTANCE;
+  private static final String KIND_VERSION = "0.11.1";
+  private static final String KUBECTL_VERSION = "1.20.6";
+  private static final Path IT_BUILD_HOME = Paths.get(System.getenv("IT_BUILD_HOME"));
+  private static final Path KUBECFG_PATH = Paths.get(IT_BUILD_HOME.toString(), "kubecfg.yml");
+  private static final Path KUBECTL_PATH = Paths.get(IT_BUILD_HOME.toString(), "kubectl");
 
-  private Path kubecfgPath;
   private final Map<String, List<String>> namespacesByAccount = new HashMap<>();
 
   public static KubernetesCluster getInstance() {
@@ -49,34 +50,38 @@ public static KubernetesCluster getInstance() {
     return INSTANCE;
   }
 
-  private KubernetesCluster() {
-    super(DOCKER_IMAGE);
-
-    // arguments to docker run
-    Map<String, String> tmpfs = new HashMap<>();
-    tmpfs.put("/run", "rw");
-    tmpfs.put("/var/run", "rw");
-    withTmpFs(tmpfs)
-        .withPrivilegedMode(true)
-        .withExposedPorts(6443)
-        .withCommand(
-            "server",
-            "--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%",
-            "--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%",
-            "--tls-san",
-            "0.0.0.0")
-        .waitingFor(Wait.forLogMessage(".*Wrote kubeconfig .*", 1));
+  private KubernetesCluster() {}
+
+  public void start() {
+    try {
+      downloadDependencies();
+      createCluster();
+    } catch (Exception e) {
+      fail("Unable to start kubernetes cluster", e);
+    }
+  }
+
+  public void stop() {
+    try {
+      runKindCmd("delete cluster --name=kube-int-tests");
+    } catch (Exception e) {
+      System.out.println("Exception deleting test cluster: " + e.getMessage() + " ignoring");
+    }
   }
 
   public Path getKubecfgPath() {
-    return kubecfgPath;
+    return KUBECFG_PATH;
   }
 
   public String createNamespace(String accountName) throws IOException, InterruptedException {
     List<String> existing =
         namespacesByAccount.computeIfAbsent(accountName, k -> new ArrayList<>());
     String newNamespace = String.format("%s-testns%02d", accountName, existing.size());
-    execKubectl("create ns " + newNamespace);
+    List<String> allNamespaces =
+        Arrays.asList(execKubectl("get ns -o=jsonpath='{.items[*].metadata.name}'").split(" "));
+    if (!allNamespaces.contains(newNamespace)) {
+      execKubectl("create ns " + newNamespace);
+    }
     existing.add(newNamespace);
     return newNamespace;
   }
@@ -92,7 +97,7 @@ public String execKubectl(String args, Map<String, Object> manifest)
     List<String> cmd = new ArrayList<>();
     cmd.add("sh");
     cmd.add("-c");
-    cmd.add("${KUBECTL_PATH} --kubeconfig=" + kubecfgPath + " " + args);
+    cmd.add(KUBECTL_PATH + " --kubeconfig=" + KUBECFG_PATH + " " + args);
     builder.command(cmd);
     builder.redirectErrorStream(true);
     Process process = builder.start();
@@ -119,48 +124,72 @@ private String manifestToJson(Map<String, Object> contents) {
     return Optional.ofNullable(contents).map(v -> new Gson().toJson(v)).orElse(null);
   }
 
-  @Override
-  public void start() {
-    super.start();
-    String containerName = getContainerInfo().getName().replaceAll("/", "");
-    System.setProperty("containername", containerName);
-    try {
-      this.kubecfgPath = copyKubecfgFromCluster(containerName);
-      fixKubeEndpoint(this.kubecfgPath);
-    } catch (IOException e) {
-      throw new RuntimeException(
-          "Unable to initialize kubectl or kubeconfig.yml files, or unable to create initial namespaces",
-          e);
+  private void downloadDependencies() throws IOException {
+    Files.createDirectories(IT_BUILD_HOME);
+    String os = "linux";
+    String arch = "amd64";
+    // TODO: Support running tests in other os/archs
+    if (System.getProperty("os.name").toLowerCase().contains("mac")) {
+      os = "darwin";
+    }
+    System.out.println("Detected os: " + os + " arch: " + arch);
+
+    Path kind = Paths.get(IT_BUILD_HOME.toString(), "kind");
+    if (!kind.toFile().exists()) {
+      String url =
+          String.format(
+              "https://github.com/kubernetes-sigs/kind/releases/download/v%s/kind-%s-%s",
+              KIND_VERSION, os, arch);
+      System.out.println("Downloading kind from " + url);
+      downloadFile(kind, url);
+    }
+
+    Path kubectl = Paths.get(IT_BUILD_HOME.toString(), "kubectl");
+    if (!kubectl.toFile().exists()) {
+      String url =
+          String.format(
+              "https://storage.googleapis.com/kubernetes-release/release/v%s/bin/%s/%s/kubectl",
+              KUBECTL_VERSION, os, arch);
+      System.out.println("Downloading kubectl from " + url);
+      downloadFile(kubectl, url);
     }
   }
 
-  @Override
-  public void stop() {
-    super.stop();
-    try {
-      Files.deleteIfExists(this.kubecfgPath);
-    } catch (IOException e) {
-      /* ignored */
+  private void downloadFile(Path binary, String url) throws IOException {
+    try (InputStream is = new URL(url).openStream();
+        ReadableByteChannel rbc = Channels.newChannel(is);
+        FileOutputStream fos = new FileOutputStream(binary.toFile())) {
+      fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
+      fos.flush();
+      assertThat(binary.toFile().setExecutable(true, false)).isEqualTo(true);
     }
   }
 
-  private Path copyKubecfgFromCluster(String containerName) throws IOException {
-    Path myKubeconfig =
-        Paths.get(System.getenv("KUBECONFIGS_HOME"), "kubecfg-" + containerName + ".yml");
-    Files.createDirectories(myKubeconfig.getParent());
-    copyFileFromContainer(KUBECFG_IN_CONTAINER, myKubeconfig.toAbsolutePath().toString());
-    return myKubeconfig;
+  private void createCluster() throws IOException, InterruptedException {
+    String clusters = runKindCmd("get clusters");
+    if (clusters.contains("kube-int-tests")) {
+      System.out.println("Deleting old test cluster");
+      runKindCmd("delete cluster --name=kube-int-tests");
+    }
+    runKindCmd("create cluster --name=kube-int-tests --kubeconfig=" + KUBECFG_PATH + " --wait=10m");
   }
 
-  @SuppressWarnings("unchecked")
-  private void fixKubeEndpoint(Path kubecfgPath) throws IOException {
-    String kubeEndpoint = "https://" + getHost() + ":" + getMappedPort(6443);
-    Yaml yaml = new Yaml();
-    InputStream inputStream = Files.newInputStream(kubecfgPath);
-    Map<String, Object> obj = yaml.load(inputStream);
-    List<Map<String, Map<String, String>>> clusters =
-        (List<Map<String, Map<String, String>>>) obj.get("clusters");
-    clusters.get(0).get("cluster").put("server", kubeEndpoint);
-    yaml.dump(obj, Files.newBufferedWriter(kubecfgPath));
+  private String runKindCmd(String args) throws IOException, InterruptedException {
+    ProcessBuilder builder = new ProcessBuilder();
+    List<String> cmd = new ArrayList<>();
+    cmd.add("sh");
+    cmd.add("-c");
+    cmd.add(Paths.get(IT_BUILD_HOME.toString(), "kind") + " " + args);
+    builder.command(cmd);
+    builder.redirectErrorStream(true);
+    Process process = builder.start();
+    Reader reader = new InputStreamReader(process.getInputStream(), UTF_8);
+    String output = FileCopyUtils.copyToString(reader);
+    System.out.println(output);
+    process.waitFor();
+    assertThat(process.exitValue())
+        .as("Running %s returned non-zero exit code. Output:\n%s", cmd, output)
+        .isEqualTo(0);
+    return output;
   }
 }

clouddriver-kubernetes/src/integration/java/com/netflix/spinnaker/clouddriver/kubernetes/it/utils/TestLifecycleListener.java

@@ -0,0 +1,36 @@
+/*
+ * Copyright 2021 Armory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.kubernetes.it.utils;
+
+import com.netflix.spinnaker.clouddriver.kubernetes.it.BaseTest;
+import org.junit.jupiter.api.extension.BeforeAllCallback;
+import org.junit.jupiter.api.extension.ExtensionContext;
+
+public class TestLifecycleListener
+    implements BeforeAllCallback, ExtensionContext.Store.CloseableResource {
+
+  @Override
+  public void beforeAll(ExtensionContext context) {
+    // initialize "after all test run hook"
+    context.getRoot().getStore(ExtensionContext.Namespace.GLOBAL).put("delete_cluster", this);
+  }
+
+  @Override
+  public void close() {
+    BaseTest.kubeCluster.stop();
+  }
+}

clouddriver-kubernetes/src/integration/resources/clouddriver.yml

@@ -4,7 +4,7 @@ spring:
 
 kubernetes:
   kubectl:
-    executable: ${KUBECTL_PATH} # Using a wrapper around the kubectl binary provided by the k3s docker container
+    executable: ${IT_BUILD_HOME}/kubectl
   enabled: true
   primaryAccount: account1
   accounts:
@@ -39,7 +39,7 @@ kubernetes:
       oAuthScopes: []
       onlySpinnakerManaged: true
       metrics: false
-      kubeconfigFile: ${KUBECONFIGS_HOME}/kubecfg-${containername}.yml  # File is automatically created during initialization
+      kubeconfigFile: ${IT_BUILD_HOME}/kubecfg.yml  # File is automatically created at runtime
     - name: account2
       cacheIntervalSeconds: 5
       requiredGroupMembership: []
@@ -66,7 +66,7 @@ kubernetes:
       oAuthScopes: []
       onlySpinnakerManaged: true
       metrics: false
-      kubeconfigFile: ${KUBECONFIGS_HOME}/kubecfg-${containername}.yml  # File is automatically created during initialization
+      kubeconfigFile: ${IT_BUILD_HOME}/kubecfg.yml  # File is automatically created at runtime
 
 logging.level.com.netflix.spinnaker.cats.sql.cluster: INFO
 logging.level.com.netflix.spinnaker.clouddriver.kubernetes.caching.agent.KubernetesCacheDataConverter: WARN