[CI] add minikubekvm integration test in prow

Author: ComradeProgrammer

Makefile

@@ -1260,3 +1260,9 @@ get-dependency-version:
 .PHONY: _update-all
 _update-all:
 	@(cd hack && go run update/update_all/update_all.go)
+
+
+
+# targets for tests on prow
+include ./hack/prow/prow.mk
+

hack/prow/common.sh

@@ -0,0 +1,200 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The script expects the following env variables:
+# OS: The operating system
+# ARCH: The architecture
+# DRIVER: the driver to use for the test
+# CONTAINER_RUNTIME: the container runtime to use for the test
+# EXTRA_START_ARGS: additional flags to pass into minikube start
+# EXTRA_TEST_ARGS: additional flags to pass into go test
+# JOB_NAME: the name of the logfile and check name to update on github
+
+function print_test_info() {
+    echo ">> Starting at $(date)"
+    echo ""
+    echo "user:      $(whoami)"
+    echo "arch:      ${OS_ARCH}"
+    echo "build:     ${MINIKUBE_LOCATION}"
+    echo "driver:    ${DRIVER}"
+    echo "runtime:   ${CONTAINER_RUNTIME}"
+    echo "job:       ${JOB_NAME}"
+    echo "test home: ${TEST_HOME}"
+    echo "kernel:    $(uname -v)"
+    echo "uptime:    $(uptime)"
+    # Setting KUBECONFIG prevents the version check from erroring out due to permission issues
+    echo "kubectl:   $(env KUBECONFIG=${TEST_HOME} kubectl version --client)"
+    echo "docker:    $(docker version --format '{{ .Client.Version }}')"
+    echo "podman:    $(sudo podman version --format '{{.Version}}' || true)"
+    echo "go:        $(go version || true)"
+
+    case "${DRIVER}" in
+    kvm2)
+        echo "virsh:     $(virsh --version)"
+        ;;
+    virtualbox)
+        echo "vbox:      $(vboxmanage --version)"
+        ;;
+    vfkit)
+        echo "vfkit:     $(vfkit --version)"
+        ;;
+    krunkit)
+        echo "krunkit:   $(krunkit --version)"
+        ;;
+    esac
+
+    echo ""
+}
+
+function install_dependencies() {
+    # We need pstree for the restart cronjobs
+    if [ "$(uname)" != "Darwin" ]; then
+        sudo apt-get -y install lsof psmisc dnsutils
+    else
+        brew install pstree coreutils pidof
+        ln -s /usr/local/bin/gtimeout /usr/local/bin/timeout || true
+    fi
+    # install golang if not present
+    sudo hack/prow/installer/check_install_golang.sh /usr/local 1.24.5 || true
+    # install gotestsum if not present
+    GOROOT="/usr/local/go" hack/prow/installer/check_install_gotestsum.sh || true
+
+
+    # install jq
+    if ! type "jq" >/dev/null; then
+        echo ">> Installing jq"
+        if [ "${ARCH}" == "arm64" && "${OS}" == "linux" ]; then
+            sudo apt-get install jq -y
+        elif [ "${ARCH}" == "arm64" ]; then
+            echo "Unable to install 'jq' automatically for arm64 on Darwin, please install 'jq' manually."
+            exit 5
+        elif [ "${OS}" != "darwin" ]; then
+            curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 &&sudo  install jq-linux64 /usr/local/bin/jq
+        else
+            curl -LO https://github.com/stedolan/jq/releases/download/jq-1.6/jq-osx-amd64 && sudo install jq-osx-amd64 /usr/local/bin/jq
+        fi
+    fi
+
+}
+
+function docker_setup() {
+
+    # clean all docker artifacts up
+    docker system prune -a --volumes -f || true
+    docker system df || true
+    docker rm -f -v $(docker ps -aq) >/dev/null 2>&1 || true
+
+    # read only token, never expires
+    #todo: do we need this token
+    # docker login -u minikubebot -p "$DOCKERHUB_READONLY_TOKEN"
+}
+
+function gvisor_image_build() {
+    # Build the gvisor image so that we can integration test changes to pkg/gvisor
+    chmod +x testdata/gvisor-addon
+    # skipping gvisor mac because ofg https://github.com/kubernetes/minikube/issues/5137
+    if [ "$(uname)" != "Darwin" ]; then
+        # Should match GVISOR_IMAGE_VERSION in Makefile
+        docker build -t gcr.io/k8s-minikube/gvisor-addon:2 -f testdata/gvisor-addon-Dockerfile ./testdata
+    fi
+}
+
+
+
+# this is where the script starts
+MINIKUBE_LOCATION="minikube_location" 
+
+readonly OS_ARCH="${OS}-${ARCH}"
+readonly TEST_ROOT="${HOME}/minikube-integration"
+readonly TEST_HOME="${TEST_ROOT}/${MINIKUBE_LOCATION}-$$"
+
+export GOPATH="$HOME/go"
+export KUBECONFIG="${TEST_HOME}/kubeconfig"
+export PATH=$PATH:"/usr/local/bin/:/usr/local/go/bin/:$GOPATH/bin"
+export MINIKUBE_SUPPRESS_DOCKER_PERFORMANCE=true
+
+readonly TIMEOUT=120m
+
+cp -r test/integration/testdata .
+cp out/gvisor-addon testdata/
+ls testdata
+
+# Add the out/ directory to the PATH, for using new drivers.
+export PATH="$(pwd)/out/":$PATH
+mkdir -p "${TEST_ROOT}"
+mkdir -p "${TEST_HOME}"
+export MINIKUBE_HOME="${TEST_HOME}/.minikube"
+export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
+export E2E_BIN="out/e2e-${OS_ARCH}"
+
+
+install_dependencies
+docker_setup
+print_test_info
+gvisor_image_build
+
+readonly TEST_OUT="${TEST_HOME}/testout.txt"
+readonly JSON_OUT="${TEST_HOME}/test.json"
+readonly JUNIT_OUT="${TEST_HOME}/junit-unit.xml"
+
+touch "${TEST_OUT}"
+touch "${JSON_OUT}"
+touch "${JUNIT_OUT}"
+
+e2e_start_time="$(date -u +%s)"
+echo ""
+echo ">> Starting ${E2E_BIN} at $(date)"
+set -x
+
+EXTRA_START_ARGS="${EXTRA_START_ARGS} --container-runtime=${CONTAINER_RUNTIME}"
+echo $PATH
+gotestsum --jsonfile "${JSON_OUT}" --junitfile="${JUNIT_OUT}" -f standard-verbose --raw-command -- \
+    go tool test2json -t \
+    ${E2E_BIN} \
+    -minikube-start-args="--driver=${DRIVER} ${EXTRA_START_ARGS}" \
+    -test.timeout=${TIMEOUT} -test.v \
+    ${EXTRA_TEST_ARGS} \
+    -binary="${MINIKUBE_BIN}" 2>&1 |
+    tee "${TEST_OUT}"
+
+result=${PIPESTATUS[0]} # capture the exit code of the first cmd in pipe.
+set +x
+echo ">> ${E2E_BIN} exited with ${result} at $(date)"
+echo ""
+
+
+
+# calculate the time took to finish running e2e binary test.
+e2e_end_time="$(date -u +%s)"
+elapsed=$(($e2e_end_time - $e2e_start_time))
+min=$(($elapsed / 60))
+sec=$(tail -c 3 <<<$((${elapsed}00 / 60)))
+elapsed=$min.$sec
+
+#todo: currently we skip gopogh upload , we shall add it back
+#upload_to_goph
+
+# according to prow's requirement, upload the test report to $ARTIFACTS
+cp ${TEST_OUT} .
+cp ${JSON_OUT} .
+cp ${JUNIT_OUT} .
+if [[ $result -eq 0 ]]; then
+    echo "minikube: SUCCESS"
+else
+    echo "minikube: FAIL"
+fi
+
+exit "$result"

hack/prow/docker.json

@@ -0,0 +1,3 @@
+{
+    "Image": "debian"
+}

hack/prow/installer/check_install_docker.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux -o pipefail
+
+ARCH=${ARCH:=amd64}
+
+
+echo "Installing latest docker"
+curl -fsSL https://get.docker.com -o get-docker.sh
+sudo sh get-docker.sh
+rm get-docker.sh
+
+sudo usermod -aG docker minitest || true
+
+echo "Installing latest kubectl"
+curl -LO "https://dl.k8s.io/release/$(curl -sL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl"
+sudo install ./kubectl /usr/local/bin/kubectl

hack/prow/installer/check_install_golang.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+# Copyright 2025 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script requires two parameters:
+# $1. INSTALL_PATH: The path to install the golang binary
+# $2. GO_VERSION: The version of golang to install
+
+
+
+set -eux -o pipefail
+
+if (($# < 2)); then
+  echo "ERROR: given ! ($#) parameters but expected 2."
+  echo "USAGE: ./check_install_golang.sh INSTALL_PATH GO_VERSION" 
+  exit 1
+fi
+
+VERSION_TO_INSTALL=${2}
+INSTALL_PATH=${1}
+
+function current_arch() {
+  case $(arch) in
+  "x86_64" | "i386")
+     echo "amd64"
+  ;;
+  "aarch64" | "arm64")
+    echo "arm64"
+  ;;
+  *)
+    echo "unexpected arch: $(arch). use amd64" 1>&2
+    echo "amd64"
+  ;;
+  esac
+}
+
+ARCH=${ARCH:=$(current_arch)}
+
+# installs or updates golang if right version doesn't exists
+function check_and_install_golang() {
+  if ! go version &>/dev/null; then
+    echo "WARNING: No golang installation found in your environment."
+    install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
+    return
+  fi
+  
+  sudo chown -R $USER:$USER "$INSTALL_PATH"/go
+
+  # golang has been installed and check its version
+  if [[ $(go version | cut -d' ' -f 3) =~ go(([0-9]+)\.([0-9]+).([0-9]+)*) ]]; then
+    HOST_VERSION=${BASH_REMATCH[1]}
+    if [ $HOST_VERSION == $VERSION_TO_INSTALL ]; then
+      echo "go version on the host looks good : $HOST_VERSION"
+    else
+      echo "WARNING: expected go version to be $VERSION_TO_INSTALL but got $HOST_VERSION"
+      install_golang "$VERSION_TO_INSTALL" "$INSTALL_PATH"
+    fi
+  else
+    echo "ERROR: Failed to parse golang version: $(go version)"
+    return
+  fi
+}
+
+# install_golang takes two parameters version and path to install.
+function install_golang() {
+  local -r GO_VER="$1"
+  local -r GO_DIR="$2/go"
+  echo "Installing golang version: $GO_VER in $GO_DIR"
+
+  INSTALLOS=linux
+  if [[ "$OSTYPE" == "darwin"* ]]; then
+    INSTALLOS=darwin
+  fi
+
+  local -r GO_TGZ="go${GO_VER}.${INSTALLOS}-${ARCH}.tar.gz"
+  pushd /tmp
+
+  # using sudo because previously installed versions might have been installed by a different user.
+  sudo rm -rf "$GO_TGZ"
+  curl -qL -O "https://storage.googleapis.com/golang/$GO_TGZ"
+  sudo rm -rf "$GO_DIR"
+  sudo mkdir -p "$GO_DIR"
+  sudo tar -C "$GO_DIR" --strip-components=1 -xzf "$GO_TGZ"
+
+  popd >/dev/null
+  echo "installed in $GO_DIR: $($GO_DIR/bin/go version)"
+}
+
+check_and_install_golang

hack/prow/installer/check_install_gotestsum.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+# Copyright 2021 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux -o pipefail
+
+function install_gotestsum() {
+  rm -f $(which gotestsum)
+   sudo env "GOBIN=$GOROOT/bin" "PATH=$PATH" go install gotest.tools/gotestsum@v1.12.3
+}
+echo "Installing gotestsum"
+which gotestsum || install_gotestsum

hack/prow/kvm.json

@@ -0,0 +1,9 @@
+{
+    "GCPZone": "us-central1-b",
+    "InstanceImage": "ubuntu-os-cloud/ubuntu-2404-lts-amd64",
+    "InstanceType": "n2-standard-32",
+    "DiskGiB": 300,
+    "BoskosAcquireTimeoutSeconds": 180,
+    "BoskosHeartbeatIntervalSeconds": 10,
+    "BoskosLocation": "http://boskos.test-pods.svc.cluster.local"
+}