diff --git a/.yetus/excludes.txt b/.yetus/excludes.txt index b2980ac886..18f9283b30 100644 --- a/.yetus/excludes.txt +++ b/.yetus/excludes.txt @@ -9,6 +9,7 @@ ^pkg/kvm-tools/include ^pkg/kvm-tools/dtrace ^pkg/kube/kubevirt-operator.yaml +^pkg/kube/multus-daemonset.yaml ^api/python/ ^api/go/ ^api/images/ diff --git a/pkg/kube/Dockerfile b/pkg/kube/Dockerfile index 6941b583d0..d45536bf72 100644 --- a/pkg/kube/Dockerfile +++ b/pkg/kube/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile-upstream:1.5.0-rc2-labs -FROM lfedge/eve-alpine:a76c418ea841a1c433e0ab562653c3801f494d8e as build -ENV BUILD_PKGS go +FROM lfedge/eve-alpine:2ce6af13cbeb90e4ba3fda8903172ecc064580bb as build +ENV BUILD_PKGS git go ENV PKGS alpine-baselayout musl-utils iproute2 iptables curl openrc \ - open-iscsi libvirt libvirt-client util-linux grep findutils jq \ - cni-plugins + open-iscsi libvirt libvirt-client util-linux grep cni-plugins \ + jq findutils nfs-utils RUN eve-alpine-deploy.sh # Remove unused CNI plugins @@ -19,29 +19,32 @@ RUN GO111MODULE=on CGO_ENABLED=0 go build -v -ldflags "-s -w" -mod=vendor -o /ou FROM scratch COPY --from=build /out/ / COPY cluster-init.sh /usr/bin/ +COPY nsmounter /usr/bin/ +COPY install-etcdctl.sh /usr/bin/ COPY cgconfig.conf /etc # kubevirt yaml files are patched files and will be removed later, look at cluster-init.sh COPY multus-daemonset.yaml /etc -COPY kubevirt-operator.yaml /etc -COPY kubevirt-features.yaml /etc RUN mkdir -p /etc/containerd +COPY kubevirt-features.yaml /etc +COPY kubevirt-operator.yaml /etc COPY config-k3s.toml /etc/containerd/ +COPY longhorn-generate-support-bundle.sh /usr/bin/ +COPY k3s-pod-logs.sh /usr/bin/ +COPY debuguser-role-binding.yaml /etc/ +COPY external-boot-image.tar /etc/ +COPY iscsid.conf /etc/iscsi/ +RUN ls -l /etc/ RUN mkdir -p /etc/rancher/k3s COPY config.yaml /etc/rancher/k3s -COPY external-boot-image.tar /etc/ WORKDIR / # Actual k3s install and config happens when this container starts during EVE bootup, look at cluster-init.sh -### NOTE: the following lines are commented out until the EVE partition size increases -#ENV VIRTCTL_VERSION v0.59.0 -#ADD https://github.com/kubevirt/kubevirt/releases/download/${VIRTCTL_VERSION}/virtctl-${VIRTCTL_VERSION}-linux-amd64 . -#RUN install virtctl-${VIRTCTL_VERSION}-linux-amd64 /usr/bin/virtctl +### NOTE: the version of virtctl should match the version of kubevirt in cluster_init.sh, else PVC creation might fail due to incompatibility +ENV VIRTCTL_VERSION v1.1.0 +ADD https://github.com/kubevirt/kubevirt/releases/download/${VIRTCTL_VERSION}/virtctl-${VIRTCTL_VERSION}-linux-amd64 . +RUN install virtctl-${VIRTCTL_VERSION}-linux-amd64 /usr/bin/virtctl # We installed under /usr/bin. Remove the downloaded version -#RUN rm -f ./virtctl-${VIRTCTL_VERSION}-linux-amd64 - -ENV ETCDCTL_VERSION v3.5.5 -ADD https://github.com/etcd-io/etcd/releases/download/${ETCDCTL_VERSION}/etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz . -RUN tar -zxv --strip-components=1 -C /usr/local/bin < ./etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz +RUN rm -f ./virtctl-${VIRTCTL_VERSION}-linux-amd64 ENTRYPOINT [] CMD ["/usr/bin/cluster-init.sh"] diff --git a/pkg/kube/cluster-init.sh b/pkg/kube/cluster-init.sh index 46c1fcf474..28f9b512b4 100755 --- a/pkg/kube/cluster-init.sh +++ b/pkg/kube/cluster-init.sh @@ -3,15 +3,19 @@ # Copyright (c) 2023 Zededa, Inc. # SPDX-License-Identifier: Apache-2.0 -K3S_VERSION=v1.26.3+k3s1 -KUBEVIRT_VERSION=v0.59.0 -LONGHORN_VERSION=v1.4.2 -CDI_VERSION=v1.56.0 -NODE_IP="" +K3S_VERSION=v1.28.5+k3s1 +KUBEVIRT_VERSION=v1.1.0 +LONGHORN_VERSION=v1.6.0 +CDI_VERSION=v1.57.0 +# shellcheck disable=SC2034 +Node_IP="" +MAX_K3S_RESTARTS=10 +RESTART_COUNT=0 +K3S_LOG_DIR="/persist/newlog/kube" +loglimitSize=$((5*1024*1024)) -INSTALL_LOG=/var/lib/install.log -CTRD_LOG=/var/lib/containerd.log -LOG_SIZE=$((5*1024*1024)) +# install log in K3S_LOG_DIR k3s-install.log +INSTALL_LOG="${K3S_LOG_DIR}/k3s-install.log" logmsg() { local MSG @@ -25,6 +29,34 @@ setup_cgroup () { echo "cgroup /sys/fs/cgroup cgroup defaults 0 0" >> /etc/fstab } +check_log_file_size() { + currentSize=$(wc -c <"$K3S_LOG_DIR/$1") + if [ "$currentSize" -lt "$loglimitSize" ]; then + return + fi + if [ -f "$K3S_LOG_DIR/$1.2" ]; then + cp "$K3S_LOG_DIR/$1.2" "$K3S_LOG_DIR/$1.3" + fi + if [ -f "$K3S_LOG_DIR/$1.1" ]; then + cp "$K3S_LOG_DIR/$1.1" "$K3S_LOG_DIR/$1.2" + fi + cp "$K3S_LOG_DIR/$1" "$K3S_LOG_DIR/$1.1" + truncate -s 0 "$K3S_LOG_DIR/$1" + logmsg "k3s logfile size $currentSize rotate" +} + +save_crash_log() { + if [ "$RESTART_COUNT" = "1" ]; then + return + fi + fileBaseName=$1 + # This pattern will alias with older crashes, but also a simple way to contain log bloat + crashLogBaseName="${fileBaseName}.restart.${RESTART_COUNT}.gz" + if [ -e "${K3S_LOG_DIR}/${crashLogBaseName}" ]; then + rm "${K3S_LOG_DIR}/${crashLogBaseName}" + fi + gzip -k -9 "${K3S_LOG_DIR}/${fileBaseName}" -c > "${K3S_LOG_DIR}/${crashLogBaseName}" +} check_network_connection () { while true; do @@ -51,6 +83,7 @@ wait_for_default_route() { return 1 } + # Get IP of the interface with the first default route. # This will be then used as K3s node IP. # XXX This is a temporary solution. Eventually, the user will be able to select @@ -70,17 +103,85 @@ get_default_intf_IP_prefix() { awk -v new_ip="$ip_prefix" '{gsub("IPAddressReplaceMe", new_ip)}1' /etc/multus-daemonset.yaml > /tmp/multus-daemonset.yaml } +# kubernetes's name must be lower case and '-' instead of '_' +convert_to_k8s_compatible() { + echo "$1" | tr '[:upper:]_' '[:lower:]-' +} + +wait_for_device_name() { + logmsg "Waiting for DeviceName from controller..." + EdgeNodeInfoPath="/persist/status/zedagent/EdgeNodeInfo/global.json" + while true; do + if [ -f $EdgeNodeInfoPath ]; then + dName=$(jq -r '.DeviceName' $EdgeNodeInfoPath) + if [ -n "$dName" ]; then + HOSTNAME=$(convert_to_k8s_compatible "$dName") + break + fi + fi + sleep 5 + done + + # we should have the uuid since we got the device name + DEVUUID=$(/bin/hostname) + # get last 5 bytes of the DEVUUID as suffix to the hostname + DEVUUID_HASH=$(echo "$DEVUUID" | tail -c 6) + HOSTNAME="$HOSTNAME-$DEVUUID_HASH" + if ! grep -q node-name /etc/rancher/k3s/config.yaml; then + echo "node-name: $HOSTNAME" >> /etc/rancher/k3s/config.yaml + fi +} + +check_start_k3s() { + pgrep -f "k3s server" > /dev/null 2>&1 + if [ $? -eq 1 ]; then + if [ $RESTART_COUNT -lt $MAX_K3S_RESTARTS ]; then + ## Must be after reboot, or from k3s restart + RESTART_COUNT=$((RESTART_COUNT+1)) + save_crash_log "k3s.log" + if [ ! -f /var/lib/cni/bin ]; then + copy_cni_plugin_files + fi + ln -s /var/lib/k3s/bin/* /usr/bin + logmsg "Starting k3s server, restart count: $RESTART_COUNT" + # for now, always copy to get the latest + nohup /usr/bin/k3s server --config /etc/rancher/k3s/config.yaml & + k3s_pid=$! + # Give the embedded etcd in k3s priority over io as its fsync latencies are critical + ionice -c2 -n0 -p $k3s_pid + # Default location where clients will look for config + while [ ! -f /etc/rancher/k3s/k3s.yaml ]; do + sleep 5 + done + ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config + cp /etc/rancher/k3s/k3s.yaml /run/.kube/k3s/k3s.yaml + sleep 10 + return 1 + else + logmsg "k3s is down and restart count exceeded." + fi + fi + return 0 +} + apply_multus_cni() { + # apply multus + sleep 10 + # get IP of the interface with the first default route, which will be used as node IP get_default_intf_IP_prefix kubectl create namespace eve-kube-app logmsg "Apply Multus, Node-IP: $NODE_IP" - while ! kubectl apply -f /tmp/multus-daemonset.yaml; do - sleep 1 - done - logmsg "Done applying Multus" + if ! kubectl apply -f /tmp/multus-daemonset.yaml; then + # Give up the cpu to the containerd/k3s restart loop + return 1 + fi + logmsg "done applying multus" ln -s /var/lib/cni/bin/multus /var/lib/rancher/k3s/data/current/bin/multus # need to only do this once + # launch CNI dhcp service + /opt/cni/bin/dhcp daemon & touch /var/lib/multus_initialized + return 0 } copy_cni_plugin_files() { @@ -109,6 +210,10 @@ mount_etcd_vol() { logmsg "Wait for persist/etcd-storage zvol" while [ ! -b /dev/zvol/persist/etcd-storage ]; do + fs=$(lsblk -f -d /dev/zvol/persist/etcd-storage | grep -v FSTYPE | awk '{print $2}') + if [ "$fs" = "ext4" ]; then + break + fi sleep 1 done mount /dev/zvol/persist/etcd-storage /var/lib ## This is where we persist the cluster components (etcd) @@ -123,6 +228,7 @@ setup_prereqs () { modprobe iscsi_tcp #Needed for iscsi tools mkdir -p /run/lock + mkdir -p "$K3S_LOG_DIR" /usr/sbin/iscsid start mount --make-rshared / setup_cgroup @@ -131,8 +237,58 @@ setup_prereqs () { check_network_connection wait_for_vault mount_etcd_vol + wait_for_device_name + chmod o+rw /dev/null } +apply_longhorn_disk_config() { + node=$1 + kubectl label node "$node" node.longhorn.io/create-default-disk='config' + kubectl annotate node "$node" node.longhorn.io/default-disks-config='[ { "path":"/persist/vault/volumes", "allowScheduling":true }]' +} + +config_cluster_roles() { + apk add openssl + # generate user debugging-user certificates + openssl genrsa -out /tmp/user.key 2048 + openssl req -new -key /tmp/user.key -out /tmp/user.csr -subj "/CN=debugging-user/O=rbac" + openssl x509 -req -in /tmp/user.csr -CA /var/lib/rancher/k3s/server/tls/client-ca.crt \ + -CAkey /var/lib/rancher/k3s/server/tls/client-ca.key -CAcreateserial -out /tmp/user.crt -days 365 + user_key_base64=$(base64 -w0 < /tmp/user.key) + user_crt_base64=$(base64 -w0 < /tmp/user.crt) + + # generate kubeConfigure user for debugging-user + cp /etc/rancher/k3s/k3s.yaml /var/lib/rancher/k3s/user.yaml + sed -i "s|client-certificate-data:.*|client-certificate-data: $user_crt_base64|g" /var/lib/rancher/k3s/user.yaml + sed -i "s|client-key-data:.*|client-key-data: $user_key_base64|g" /var/lib/rancher/k3s/user.yaml + cp /var/lib/rancher/k3s/user.yaml /run/.kube/k3s/user.yaml + + # apply kubernetes and kubevirt roles and binding to debugging-user + kubectl apply -f /etc/debuguser-role-binding.yaml + touch /var/lib/debuguser-initialized +} + +check_overwrite_nsmounter() { + ### REMOVE ME+ + # When https://github.com/longhorn/longhorn/issues/6857 is resolved, remove this 'REMOVE ME' section + # In addition to pkg/kube/nsmounter and the copy of it in pkg/kube/Dockerfile + longhornCsiPluginPods=$(kubectl -n longhorn-system get pod -o json | jq -r '.items[] | select(.metadata.labels.app=="longhorn-csi-plugin" and .status.phase=="Running") | .metadata.name') + for csiPod in $longhornCsiPluginPods; do + if ! kubectl -n longhorn-system exec "pod/${csiPod}" --container=longhorn-csi-plugin -- ls /usr/local/sbin/nsmounter.updated > /dev/null 2>@1; then + if kubectl -n longhorn-system exec -i "pod/${csiPod}" --container=longhorn-csi-plugin -- tee /usr/local/sbin/nsmounter < /usr/bin/nsmounter; then + logmsg "Updated nsmounter in longhorn pod ${csiPod}" + kubectl -n longhorn-system exec "pod/${csiPod}" --container=longhorn-csi-plugin -- touch /usr/local/sbin/nsmounter.updated + fi + fi + done + ### REMOVE ME- +} + +HOSTNAME="" +DEVUUID="" +#Make sure all prereqs are set after /var/lib is mounted to get logging info +setup_prereqs + check_start_containerd() { # Needed to get the pods to start if [ ! -L /usr/bin/runc ]; then @@ -142,32 +298,31 @@ check_start_containerd() { ln -s /var/lib/rancher/k3s/data/current/bin/containerd-shim-runc-v2 /usr/bin/containerd-shim-runc-v2 fi - if pgrep -f "containerd --config" >> $INSTALL_LOG 2>&1; then - logmsg "k3s-containerd is alive" - else - logmsg "Starting k3s-containerd" + pgrep -f "/var/lib/rancher/k3s/data/current/bin/containerd" > /dev/null 2>&1 + if [ $? -eq 1 ]; then mkdir -p /run/containerd-user - nohup /var/lib/rancher/k3s/data/current/bin/containerd --config /etc/containerd/config-k3s.toml > $CTRD_LOG 2>&1 & + nohup /var/lib/rancher/k3s/data/current/bin/containerd --config /etc/containerd/config-k3s.toml & + containerd_pid=$! + logmsg "Started k3s-containerd at pid:$containerd_pid" fi if [ -f /etc/external-boot-image.tar ]; then # NOTE: https://kubevirt.io/user-guide/virtual_machines/boot_from_external_source/ # Install external-boot-image image to our eve user containerd registry. # This image contains just kernel and initrd to bootstrap a container image as a VM. # This is very similar to what we do on kvm based eve to start container as a VM. - logmsg "Trying to install new external-boot-image" + logmsg "trying to install new external-boot-image" # This import happens once per reboot - ctr -a /run/containerd-user/containerd.sock image import /etc/external-boot-image.tar docker.io/lfedge/eve-external-boot-image:latest - res=$? - if [ $res -eq 0 ]; then + if ctr -a /run/containerd-user/containerd.sock image import /etc/external-boot-image.tar docker.io/lfedge/eve-external-boot-image:latest; then logmsg "Successfully installed external-boot-image" rm -f /etc/external-boot-image.tar fi fi } + trigger_k3s_selfextraction() { - # Analysis of the k3s source shows nearly any cli command will first self-extract a series of binaries. - # In our case we're looking for the containerd binary. - # k3s check-config appears to be the only cli cmd which doesn't: + # This is extracted when k3s server first starts + # analysis of the k3s source shows any cli command will first extract the binaries. + # so we'll just run one, check-config appears to be the only one which doesn't: # - start a long running process/server # - timeout connecting to a socket # - manipulate config/certs @@ -183,72 +338,210 @@ trigger_k3s_selfextraction() { /usr/bin/k3s check-config >> $INSTALL_LOG 2>&1 } +# wait for debugging flag in /persist/k3s/wait_{flagname} if exist +wait_for_item() { + filename="/persist/k3s/wait_$1" + processname="k3s server" + while true; do + if [ -e "$filename" ]; then + k3sproc="" + if pgrep -x "$processname" > /dev/null; then + k3sproc="k3s server is running" + else + k3sproc="k3s server is NOT running" + fi + logmsg "Found $filename file. $k3sproc, Waiting for 60 seconds..." + sleep 60 + else + break + fi + done +} -#Make sure all prereqs are set after /var/lib is mounted to get logging info -setup_prereqs +check_node_ready_k3s_running() { + # Function to check if 'k3s server' process is running + check_k3s_running() { + pgrep -x "k3s server" > /dev/null + } + + # Function to check if the Kubernetes node is ready + check_node_ready() { + kubectl get node | grep "$HOSTNAME" | awk '{print $2}' | grep 'Ready' + } + + # Continuously check both conditions + while true; do + if ! check_k3s_running; then + # shellcheck disable=SC2034 + K3S_RUNNING=false + logmsg "k3s server is not running, exit wait" + break + fi + + if check_node_ready; then + logmsg "set node label with uuid $DEVUUID" + kubectl label node "$HOSTNAME" node-uuid="$DEVUUID" + break + fi -date >> $INSTALL_LOG + # Sleep for a while before checking again + logmsg "wait 10 more sec for node to be ready on $HOSTNAME" + sleep 10 + done +} + +# Return success if all pods in existence are Running/Succeeded and Ready +# Return unix style 0 for success. (Not 0 for false) +are_all_pods_ready() { + not_running=$(kubectl get pods -A -o json | jq '.items[] | select(.status.phase!="Running" and .status.phase!="Succeeded")' | jq -s length) + if [ "$not_running" -ne 0 ]; then + return 1 + fi + not_ready=$(kubectl get pods -A -o json | jq '.items[] | select(.status.phase=="Running") | .status.conditions[] | select(.type=="ContainersReady" and .status!="True")' | jq -s length) + if [ "$not_ready" -ne 0 ]; then + return 1 + fi + return 0 +} + +VMICONFIG_FILENAME="/run/zedkube/vmiVNC.run" +VNC_RUNNING=false +# run virtctl vnc +check_and_run_vnc() { + pid=$(pgrep -f "/usr/bin/virtctl vnc" ) + # if remote-console config file exist, and either has not started, or need to restart + if [ -f "$VMICONFIG_FILENAME" ] && { [ "$VNC_RUNNING" = false ] || [ -z "$pid" ]; } then + vmiName="" + vmiPort="" + + # Read the file and extract values + while IFS= read -r line; do + # shellcheck disable=SC3010 + if [[ $line == *"VMINAME:"* ]]; then + vmiName="${line#*VMINAME:}" # Extract the part after "VMINAME:" + vmiName="${vmiName%%[[:space:]]*}" # Remove leading/trailing whitespace + elif [[ $line == *"VNCPORT:"* ]]; then + vmiPort="${line#*VNCPORT:}" # Extract the part after "VNCPORT:" + vmiPort="${vmiPort%%[[:space:]]*}" # Remove leading/trailing whitespace + fi + done < "$VMICONFIG_FILENAME" + + # Check if vminame and vncport were found and assign default values if not + if [ -z "$vmiName" ] || [ -z "$vmiPort" ]; then + logmsg "Error: VMINAME or VNCPORT is empty in $VMICONFIG_FILENAME" + return 1 + fi + + logmsg "virctl vnc on vmiName: $vmiName, port $vmiPort" + nohup /usr/bin/virtctl vnc "$vmiName" -n eve-kube-app --port "$vmiPort" --proxy-only & + VNC_RUNNING=true + else + if [ ! -f "$VMICONFIG_FILENAME" ]; then + if [ "$VNC_RUNNING" = true ]; then + if [ -n "$pid" ]; then + logmsg "Killing process with PID $pid" + kill -9 "$pid" + else + logmsg "Error: Process not found" + fi + fi + VNC_RUNNING=false + fi + fi +} + +## before that, the logs in install.log will not be shown +DATESTR=$(date) +echo "========================== $DATESTR ==========================" >> $INSTALL_LOG +echo "cluster-init.sh start for $HOSTNAME, uuid $DEVUUID" >> $INSTALL_LOG +logmsg "Using ZFS persistent storage" #Forever loop every 15 secs while true; do if [ ! -f /var/lib/all_components_initialized ]; then if [ ! -f /var/lib/k3s_initialized ]; then - #/var/lib is where all kubernetes components get installed. - logmsg "Installing K3S version $K3S_VERSION" + logmsg "Installing K3S version $K3S_VERSION on $HOSTNAME" mkdir -p /var/lib/k3s/bin /usr/bin/curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=${K3S_VERSION} INSTALL_K3S_SKIP_ENABLE=true INSTALL_K3S_BIN_DIR=/var/lib/k3s/bin sh - - ln -s /var/lib/k3s/bin/* /usr/bin + sleep 5 logmsg "Initializing K3S version $K3S_VERSION" + ln -s /var/lib/k3s/bin/* /usr/bin trigger_k3s_selfextraction - check_start_containerd - nohup /usr/bin/k3s server --config /etc/rancher/k3s/config.yaml & - #wait until k3s is ready - logmsg "Looping until k3s is ready" - while [ "$(kubectl get node "$(/bin/hostname)" -o json | jq '.status.conditions[] | select(.reason=="KubeletReady") | .status=="True"')" != "true" ]; - do - sleep 5; - done - # Give the embedded etcd in k3s priority over io as its fsync latencies are critical - ionice -c2 -n0 -p "$(pgrep -f "k3s server")" - logmsg "k3s is ready on this node" - # Default location where clients will look for config - ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config touch /var/lib/k3s_initialized fi - if [ ! -f /var/lib/cni/bin ]; then - copy_cni_plugin_files + check_start_containerd + check_start_k3s + + node_count_ready=$(kubectl get node | grep -w "$HOSTNAME" | grep -cw Ready ) + if [ "$node_count_ready" -ne 1 ]; then + sleep 10 + continue + fi + node_uuid_len=$(kubectl get nodes -l node-uuid="$DEVUUID" -o json | jq '.items | length') + if [ "$node_uuid_len" -eq 0 ]; then + logmsg "set node label with uuid $DEVUUID" + kubectl label node "$HOSTNAME" node-uuid="$DEVUUID" fi + if [ ! -f /var/lib/multus_initialized ]; then + logmsg "Installing multus cni" apply_multus_cni + sleep 10 + continue fi - if ! pidof dhcp; then - # launch CNI dhcp service - /opt/cni/bin/dhcp daemon & + + # setup debug user credential, role and binding + if [ ! -f /var/lib/debuguser-initialized ]; then + if are_all_pods_ready; then + config_cluster_roles + fi + sleep 10 + continue fi if [ ! -f /var/lib/kubevirt_initialized ]; then - # This patched version will be removed once the following PR https://github.com/kubevirt/kubevirt/pull/9668 is merged - logmsg "Installing patched Kubevirt" - kubectl apply -f /etc/kubevirt-operator.yaml - kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml - - #CDI (containerzed data importer) is need to convert qcow2/raw formats to Persistent Volumes and Data volumes - #Since CDI goes with kubevirt we install with that. - logmsg "Installing CDI version $CDI_VERSION" - kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$CDI_VERSION/cdi-operator.yaml - kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$CDI_VERSION/cdi-cr.yaml - #Add kubevirt feature gates - kubectl apply -f /etc/kubevirt-features.yaml - - touch /var/lib/kubevirt_initialized + if are_all_pods_ready; then + wait_for_item "kubevirt" + # This patched version will be removed once the following PR https://github.com/kubevirt/kubevirt/pull/9668 is merged + logmsg "Installing patched Kubevirt" + kubectl apply -f /etc/kubevirt-operator.yaml + kubectl apply -f https://github.com/kubevirt/kubevirt/releases/download/${KUBEVIRT_VERSION}/kubevirt-cr.yaml + + wait_for_item "cdi" + #CDI (containerzed data importer) is need to convert qcow2/raw formats to Persistent Volumes and Data volumes + #Since CDI goes with kubevirt we install with that. + logmsg "Installing CDI version $CDI_VERSION" + kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$CDI_VERSION/cdi-operator.yaml + kubectl create -f https://github.com/kubevirt/containerized-data-importer/releases/download/$CDI_VERSION/cdi-cr.yaml + + #Add feature gates + kubectl apply -f /etc/kubevirt-features.yaml + + touch /var/lib/kubevirt_initialized + fi + sleep 10 + continue fi if [ ! -f /var/lib/longhorn_initialized ]; then - logmsg "Installing longhorn version ${LONGHORN_VERSION}" - kubectl apply -f https://raw.githubusercontent.com/longhorn/longhorn/${LONGHORN_VERSION}/deploy/longhorn.yaml - touch /var/lib/longhorn_initialized + if are_all_pods_ready; then + wait_for_item "longhorn" + logmsg "Installing longhorn version ${LONGHORN_VERSION}" + apply_longhorn_disk_config "$HOSTNAME" + lhCfgPath=/var/lib/lh-cfg-${LONGHORN_VERSION}.yaml + if [ ! -e $lhCfgPath ]; then + curl -k https://raw.githubusercontent.com/longhorn/longhorn/${LONGHORN_VERSION}/deploy/longhorn.yaml > "$lhCfgPath" + fi + if ! grep -q 'create-default-disk-labeled-nodes: true' "$lhCfgPath"; then + sed -i '/ default-setting.yaml: |-/a\ create-default-disk-labeled-nodes: true' "$lhCfgPath" + fi + kubectl apply -f "$lhCfgPath" + touch /var/lib/longhorn_initialized + fi + sleep 10 + continue fi if [ -f /var/lib/k3s_initialized ] && [ -f /var/lib/kubevirt_initialized ] && [ -f /var/lib/longhorn_initialized ]; then @@ -257,41 +550,30 @@ if [ ! -f /var/lib/all_components_initialized ]; then fi else check_start_containerd - if pgrep k3s >> $INSTALL_LOG 2>&1; then - logmsg "k3s is alive " - else - ## Must be after reboot - ln -s /var/lib/k3s/bin/* /usr/bin - logmsg "Starting k3s server after reboot" - nohup /usr/bin/k3s server --config /etc/rancher/k3s/config.yaml & - logmsg "Looping until k3s is ready" - while [ "$(kubectl get node "$(/bin/hostname)" -o json | jq '.status.conditions[] | select(.reason=="KubeletReady") | .status=="True"')" != "true" ]; - do - sleep 5; - done - # Give the embedded etcd in k3s priority over io as its fsync latencies are critical - ionice -c2 -n0 -p "$(pgrep -f "k3s server")" - logmsg "k3s is ready on this node" - # Default location where clients will look for config - ln -s /etc/rancher/k3s/k3s.yaml ~/.kube/config - - # Initialize CNI after k3s reboot - if [ ! -f /var/lib/cni/bin ]; then - copy_cni_plugin_files - fi + if ! check_start_k3s; then if [ ! -f /var/lib/multus_initialized ]; then apply_multus_cni fi - if ! pidof dhcp; then - # launch CNI dhcp service - /opt/cni/bin/dhcp daemon & + # launch CNI dhcp service + /opt/cni/bin/dhcp daemon & + + # setup debug user credential, role and binding + if [ ! -f /var/lib/debuguser-initialized ]; then + config_cluster_roles + else + cp /var/lib/rancher/k3s/user.yaml /run/.kube/k3s/user.yaml + fi + else + if [ -e /var/lib/longhorn_initialized ]; then + check_overwrite_nsmounter fi fi fi - currentSize=$(wc -c <"$CTRD_LOG") - if [ "$currentSize" -gt "$LOG_SIZE" ]; then - cp "$CTRD_LOG" "${CTRD_LOG}.1" - truncate -s 0 "$CTRD_LOG" - fi - sleep 15 + check_log_file_size "k3s.log" + check_log_file_size "multus.log" + check_log_file_size "k3s-install.log" + check_log_file_size "eve-bridge.log" + check_and_run_vnc + wait_for_item "wait" + sleep 30 done diff --git a/pkg/kube/config-k3s.toml b/pkg/kube/config-k3s.toml index 6f4b65e099..a95d2f1007 100644 --- a/pkg/kube/config-k3s.toml +++ b/pkg/kube/config-k3s.toml @@ -3,9 +3,10 @@ version = 2 state = "/run/containerd-user" +root = "/persist/vault/containerd" [plugins."io.containerd.internal.v1.opt"] - path = "/var/lib/rancher/k3s/agent/containerd/" + path = "/persist/vault/containerd" [plugins."io.containerd.grpc.v1.cri"] stream_server_address = "127.0.0.1" stream_server_port = "10010" @@ -22,7 +23,7 @@ state = "/run/containerd-user" address = "/run/containerd-user/containerd.sock" [plugins."io.containerd.grpc.v1.cri".cni] - bin_dir = "/var/lib/rancher/k3s/data/c26e7571d760c5f199d18efd197114f1ca4ab1e6ffe494f96feb65c87fcb8cf0/bin" + bin_dir = "/var/lib/rancher/k3s/data/current/bin" conf_dir = "/var/lib/rancher/k3s/agent/etc/cni/net.d" [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] @@ -32,4 +33,3 @@ state = "/run/containerd-user" SystemdCgroup = false - diff --git a/pkg/kube/config.yaml b/pkg/kube/config.yaml index 893904e399..14d3af105f 100644 --- a/pkg/kube/config.yaml +++ b/pkg/kube/config.yaml @@ -2,12 +2,14 @@ # k3s server config file. write-kubeconfig-mode: "0644" -cluster-init: true -log: "/var/lib/rancher/k3s/k3s.log" -# Remove debug flag before release to avoid overlogging -debug: true -etcd-expose-metrics: true -container-runtime-endpoint: "/run/containerd-user/containerd.sock" +log: "/persist/newlog/kube/k3s.log" +# Use longhorn storage +disable: local-storage etcd-arg: + - "heartbeat-interval=8000" + - "election-timeout=40000" - "quota-backend-bytes=8589934592" -disable-network-policy: true \ No newline at end of file +etcd-expose-metrics: true +container-runtime-endpoint: "/run/containerd-user/containerd.sock" +disable-network-policy: true +disable-cloud-controller: true diff --git a/pkg/kube/debuguser-role-binding.yaml b/pkg/kube/debuguser-role-binding.yaml new file mode 100644 index 0000000000..0a4965c4fb --- /dev/null +++ b/pkg/kube/debuguser-role-binding.yaml @@ -0,0 +1,112 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: vm-read-access + labels: + kubevirt.io: "" +rules: + - apiGroups: + - subresources.kubevirt.io + resources: + - virtualmachineinstances/vnc + - virtualmachineinstances/console + verbs: + - get + - apiGroups: + - kubevirt.io + resources: + - kubevirts + - virtualmachines + - virtualmachineinstances + - virtualmachineinstancepresets + - virtualmachineinstancereplicasets + verbs: + - get + - list + - apiGroups: + - longhorn.io + resources: + - volumes + - volumes/status + - replicas + - replicas/status + - nodes + - nodes/status + verbs: + - get + - list + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - apiGroups: + - metrics.k8s.io + resources: + - nodes + - pods + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: vm-read-access-binding +subjects: + - kind: User + name: debugging-user + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: vm-read-access + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: debugging-cluster-role +rules: + - apiGroups: ["", "apps", "extensions", "k8s.cni.cncf.io", "apiextensions.k8s.io"] + resources: ['*'] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: debugging-cluster-role-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: debugging-cluster-role +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: debugging-user +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: debugging-exec-role + namespace: eve-kube-app +rules: + - apiGroups: [""] + resources: ['pods/exec'] + verbs: ["create"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: debugging-exec-role-binding + namespace: eve-kube-app +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: debugging-exec-role +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: User + name: debugging-user diff --git a/pkg/kube/install-etcdctl.sh b/pkg/kube/install-etcdctl.sh new file mode 100755 index 0000000000..b98dfa2864 --- /dev/null +++ b/pkg/kube/install-etcdctl.sh @@ -0,0 +1,9 @@ +#!/bin/sh +# +# Copyright (c) 2023 Zededa, Inc. +# SPDX-License-Identifier: Apache-2.0 + +ETCDCTL_VERSION=v3.5.5 +/usr/bin/wget https://github.com/etcd-io/etcd/releases/download/${ETCDCTL_VERSION}/etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz +tar -zxv --strip-components=1 -C /usr/local/bin < ./etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz +rm ./etcd-${ETCDCTL_VERSION}-linux-amd64.tar.gz diff --git a/pkg/kube/iscsid.conf b/pkg/kube/iscsid.conf new file mode 100644 index 0000000000..64a1927a1b --- /dev/null +++ b/pkg/kube/iscsid.conf @@ -0,0 +1,337 @@ +# +# Open-iSCSI default configuration. +# Could be located at /etc/iscsi/iscsid.conf or ~/.iscsid.conf +# +# Note: To set any of these values for a specific node/session run +# the iscsiadm --mode node --op command for the value. See the README +# and man page for iscsiadm for details on the --op command. +# + +###################### +# iscsid daemon config +###################### +# +# If you want iscsid to start the first time an iscsi tool +# needs to access it, instead of starting it when the init +# scripts run, set the iscsid startup command here. This +# should normally only need to be done by distro package +# maintainers. If you leave the iscsid daemon running all +# the time then leave this attribute commented out. +# + +# Alpine startup + iscsid.startup = /etc/init.d/iscsid start +# +# Default for Fedora and RHEL. (uncomment to activate). +# iscsid.startup = /bin/systemctl start iscsid.socket iscsiuio.socket +# +# Default if you are not using systemd (uncomment to activate) +# iscsid.startup = /usr/bin/service start iscsid + +# Check for active mounts on devices reachable through a session +# and refuse to logout if there are any. Defaults to "No". +# iscsid.safe_logout = Yes + +############################# +# NIC/HBA and driver settings +############################# +# open-iscsi can create a session and bind it to a NIC/HBA. +# To set this up see the example iface config file. + +#***************** +# Startup settings +#***************** + +# To request that the iscsi initd scripts startup a session set to "automatic". +# node.startup = automatic +# +# To manually startup the session set to "manual". The default is manual. +node.startup = manual + +# For "automatic" startup nodes, setting this to "Yes" will try logins on each +# available iface until one succeeds, and then stop. The default "No" will try +# logins on all available ifaces simultaneously. +node.leading_login = No + +# ************* +# CHAP Settings +# ************* + +# To enable CHAP authentication set node.session.auth.authmethod +# to CHAP. The default is None. +#node.session.auth.authmethod = CHAP + +# To configure which CHAP algorithms to enable set +# node.session.auth.chap_algs to a comma separated list. +# The algorithms should be listen with most preferred first. +# Valid values are MD5, SHA1, SHA256, and SHA3-256. +# The default is MD5. +#node.session.auth.chap_algs = SHA3-256,SHA256,SHA1,MD5 + +# To set a CHAP username and password for initiator +# authentication by the target(s), uncomment the following lines: +#node.session.auth.username = username +#node.session.auth.password = password + +# To set a CHAP username and password for target(s) +# authentication by the initiator, uncomment the following lines: +#node.session.auth.username_in = username_in +#node.session.auth.password_in = password_in + +# To enable CHAP authentication for a discovery session to the target +# set discovery.sendtargets.auth.authmethod to CHAP. The default is None. +#discovery.sendtargets.auth.authmethod = CHAP + +# To set a discovery session CHAP username and password for the initiator +# authentication by the target(s), uncomment the following lines: +#discovery.sendtargets.auth.username = username +#discovery.sendtargets.auth.password = password + +# To set a discovery session CHAP username and password for target(s) +# authentication by the initiator, uncomment the following lines: +#discovery.sendtargets.auth.username_in = username_in +#discovery.sendtargets.auth.password_in = password_in + +# ******** +# Timeouts +# ******** +# +# See the iSCSI README's Advanced Configuration section for tips +# on setting timeouts when using multipath or doing root over iSCSI. +# +# To specify the length of time to wait for session re-establishment +# before failing SCSI commands back to the application when running +# the Linux SCSI Layer error handler, edit the line. +# The value is in seconds and the default is 120 seconds. +# Special values: +# - If the value is 0, IO will be failed immediately. +# - If the value is less than 0, IO will remain queued until the session +# is logged back in, or until the user runs the logout command. +node.session.timeo.replacement_timeout = 120 + +# To specify the time to wait for login to complete, edit the line. +# The value is in seconds and the default is 15 seconds. +node.conn[0].timeo.login_timeout = 15 + +# To specify the time to wait for logout to complete, edit the line. +# The value is in seconds and the default is 15 seconds. +node.conn[0].timeo.logout_timeout = 15 + +# Time interval to wait for on connection before sending a ping. +node.conn[0].timeo.noop_out_interval = 5 + +# To specify the time to wait for a Nop-out response before failing +# the connection, edit this line. Failing the connection will +# cause IO to be failed back to the SCSI layer. If using dm-multipath +# this will cause the IO to be failed to the multipath layer. +node.conn[0].timeo.noop_out_timeout = 5 + +# To specify the time to wait for abort response before +# failing the operation and trying a logical unit reset edit the line. +# The value is in seconds and the default is 15 seconds. +node.session.err_timeo.abort_timeout = 30 + +# To specify the time to wait for a logical unit response +# before failing the operation and trying session re-establishment +# edit the line. +# The value is in seconds and the default is 30 seconds. +node.session.err_timeo.lu_reset_timeout = 60 + +# To specify the time to wait for a target response +# before failing the operation and trying session re-establishment +# edit the line. +# The value is in seconds and the default is 30 seconds. +node.session.err_timeo.tgt_reset_timeout = 60 + + +#****** +# Retry +#****** + +# To specify the number of times iscsid should retry a login +# if the login attempt fails due to the node.conn[0].timeo.login_timeout +# expiring modify the following line. Note that if the login fails +# quickly (before node.conn[0].timeo.login_timeout fires) because the network +# layer or the target returns an error, iscsid may retry the login more than +# node.session.initial_login_retry_max times. +# +# This retry count along with node.conn[0].timeo.login_timeout +# determines the maximum amount of time iscsid will try to +# establish the initial login. node.session.initial_login_retry_max is +# multiplied by the node.conn[0].timeo.login_timeout to determine the +# maximum amount. +# +# The default node.session.initial_login_retry_max is 8 and +# node.conn[0].timeo.login_timeout is 15 so we have: +# +# node.conn[0].timeo.login_timeout * node.session.initial_login_retry_max = +# 120 seconds +# +# Valid values are any integer value. This only +# affects the initial login. Setting it to a high value can slow +# down the iscsi service startup. Setting it to a low value can +# cause a session to not get logged into, if there are distuptions +# during startup or if the network is not ready at that time. +node.session.initial_login_retry_max = 8 + +################################ +# session and device queue depth +################################ + +# To control how many commands the session will queue set +# node.session.cmds_max to an integer between 2 and 2048 that is also +# a power of 2. The default is 128. +node.session.cmds_max = 128 + +# To control the device's queue depth set node.session.queue_depth +# to a value between 1 and 1024. The default is 32. +node.session.queue_depth = 1 + +################################## +# MISC SYSTEM PERFORMANCE SETTINGS +################################## + +# For software iscsi (iscsi_tcp) and iser (ib_iser) each session +# has a thread used to transmit or queue data to the hardware. For +# cxgb3i you will get a thread per host. +# +# Setting the thread's priority to a lower value can lead to higher throughput +# and lower latencies. The lowest value is -20. Setting the priority to +# a higher value, can lead to reduced IO performance, but if you are seeing +# the iscsi or scsi threads dominate the use of the CPU then you may want +# to set this value higher. +# +# Note: For cxgb3i you must set all sessions to the same value, or the +# behavior is not defined. +# +# The default value is -20. The setting must be between -20 and 20. +node.session.xmit_thread_priority = -20 + + +#*************** +# iSCSI settings +#*************** + +# To enable R2T flow control (i.e., the initiator must wait for an R2T +# command before sending any data), uncomment the following line: +# +#node.session.iscsi.InitialR2T = Yes +# +# To disable R2T flow control (i.e., the initiator has an implied +# initial R2T of "FirstBurstLength" at offset 0), uncomment the following line: +# +# The defaults is No. +node.session.iscsi.InitialR2T = No + +# +# To disable immediate data (i.e., the initiator does not send +# unsolicited data with the iSCSI command PDU), uncomment the following line: +# +#node.session.iscsi.ImmediateData = No +# +# To enable immediate data (i.e., the initiator sends unsolicited data +# with the iSCSI command packet), uncomment the following line: +# +# The default is Yes +node.session.iscsi.ImmediateData = Yes + +# To specify the maximum number of unsolicited data bytes the initiator +# can send in an iSCSI PDU to a target, edit the following line. +# +# The value is the number of bytes in the range of 512 to (2^24-1) and +# the default is 262144 +node.session.iscsi.FirstBurstLength = 262144 + +# To specify the maximum SCSI payload that the initiator will negotiate +# with the target for, edit the following line. +# +# The value is the number of bytes in the range of 512 to (2^24-1) and +# the default is 16776192 +node.session.iscsi.MaxBurstLength = 16776192 + +# To specify the maximum number of data bytes the initiator can receive +# in an iSCSI PDU from a target, edit the following line. +# +# The value is the number of bytes in the range of 512 to (2^24-1) and +# the default is 262144 +node.conn[0].iscsi.MaxRecvDataSegmentLength = 262144 + +# To specify the maximum number of data bytes the initiator will send +# in an iSCSI PDU to the target, edit the following line. +# +# The value is the number of bytes in the range of 512 to (2^24-1). +# Zero is a special case. If set to zero, the initiator will use +# the target's MaxRecvDataSegmentLength for the MaxXmitDataSegmentLength. +# The default is 0. +node.conn[0].iscsi.MaxXmitDataSegmentLength = 0 + +# To specify the maximum number of data bytes the initiator can receive +# in an iSCSI PDU from a target during a discovery session, edit the +# following line. +# +# The value is the number of bytes in the range of 512 to (2^24-1) and +# the default is 32768 +discovery.sendtargets.iscsi.MaxRecvDataSegmentLength = 32768 + +# To allow the targets to control the setting of the digest checking, +# with the initiator requesting a preference of enabling the checking, uncomment# one or both of the following lines: +#node.conn[0].iscsi.HeaderDigest = CRC32C,None +#node.conn[0].iscsi.DataDigest = CRC32C,None +# +# To allow the targets to control the setting of the digest checking, +# with the initiator requesting a preference of disabling the checking, +# uncomment one or both of the following lines: +#node.conn[0].iscsi.HeaderDigest = None,CRC32C +#node.conn[0].iscsi.DataDigest = None,CRC32C +# +# To enable CRC32C digest checking for the header and/or data part of +# iSCSI PDUs, uncomment one or both of the following lines: +#node.conn[0].iscsi.HeaderDigest = CRC32C +#node.conn[0].iscsi.DataDigest = CRC32C +# +# To disable digest checking for the header and/or data part of +# iSCSI PDUs, uncomment one or both of the following lines: +#node.conn[0].iscsi.HeaderDigest = None +#node.conn[0].iscsi.DataDigest = None +# +# The default is to never use DataDigests or HeaderDigests. +# + +# For multipath configurations, you may want more than one session to be +# created on each iface record. If node.session.nr_sessions is greater +# than 1, performing a 'login' for that node will ensure that the +# appropriate number of sessions is created. +node.session.nr_sessions = 1 + +# When iscsid starts up it recovers existing sessions, if possible. +# If the target for a session has gone away when this occurs, the +# iscsid daemon normally tries to reestablish each session, +# in succession, in the background, by trying again every two +# seconds, until all sessions are restored. This configuration +# variable can limits the number of retries for each session. +# For example, setting reopen_max=150 would mean that each session +# recovery was limited to about five minutes. +node.session.reopen_max = 0 + +#************ +# Workarounds +#************ + +# Some targets like IET prefer after an initiator has sent a task +# management function like an ABORT TASK or LOGICAL UNIT RESET, that +# it does not respond to PDUs like R2Ts. To enable this behavior uncomment +# the following line (The default behavior is Yes): +node.session.iscsi.FastAbort = Yes + +# Some targets like Equalogic prefer that after an initiator has sent +# a task management function like an ABORT TASK or LOGICAL UNIT RESET, that +# it continue to respond to R2Ts. To enable this uncomment this line +# node.session.iscsi.FastAbort = No + +# To prevent doing automatic scans that would add unwanted luns to the system +# we can disable them and have sessions only do manually requested scans. +# Automatic scans are performed on startup, on login, and on AEN/AER reception +# on devices supporting it. For HW drivers all sessions will use the value +# defined in the configuration file. This configuration option is independent +# of scsi_mod scan parameter. (The default behavior is auto): +node.session.scan = auto diff --git a/pkg/kube/k3s-pod-logs.sh b/pkg/kube/k3s-pod-logs.sh new file mode 100755 index 0000000000..e57ee36986 --- /dev/null +++ b/pkg/kube/k3s-pod-logs.sh @@ -0,0 +1,5 @@ +#!/bin/sh +rm -f /persist/newlog/kube/k3s-pod-logs-*.tar.gz +OUT_FILE=/persist/newlog/kube/k3s-pod-logs-$(date +'%Y%m%d-%H%M%S' -u).tar.gz +tar cfz "$OUT_FILE" -C /var/log/pods/ . +echo "Created: $OUT_FILE" \ No newline at end of file diff --git a/pkg/kube/kubevirt-features.yaml b/pkg/kube/kubevirt-features.yaml index 1950750bf2..732f4e019f 100644 --- a/pkg/kube/kubevirt-features.yaml +++ b/pkg/kube/kubevirt-features.yaml @@ -1,3 +1,4 @@ +--- apiVersion: kubevirt.io/v1 kind: KubeVirt metadata: @@ -6,8 +7,9 @@ metadata: spec: configuration: permittedHostDevices: - pciHostDevices: # <- PCIe passthrough devices like nvme drives + pciHostDevices: # <- PCIe passthrough devices like nvme drives/NIC mediatedDevices: # <- GPUs + usb: # <- USB passthrough developerConfiguration: featureGates: - LiveMigration diff --git a/pkg/kube/kubevirt-operator.yaml b/pkg/kube/kubevirt-operator.yaml index d8578c3742..ed2dd1d7a0 100644 --- a/pkg/kube/kubevirt-operator.yaml +++ b/pkg/kube/kubevirt-operator.yaml @@ -139,6 +139,95 @@ spec: type: object type: object type: object + architectureConfiguration: + properties: + amd64: + properties: + emulatedMachines: + items: + type: string + type: array + x-kubernetes-list-type: atomic + machineType: + type: string + ovmfPath: + type: string + type: object + arm64: + properties: + emulatedMachines: + items: + type: string + type: array + x-kubernetes-list-type: atomic + machineType: + type: string + ovmfPath: + type: string + type: object + defaultArchitecture: + type: string + ppc64le: + properties: + emulatedMachines: + items: + type: string + type: array + x-kubernetes-list-type: atomic + machineType: + type: string + ovmfPath: + type: string + type: object + type: object + autoCPULimitNamespaceLabelSelector: + description: When set, AutoCPULimitNamespaceLabelSelector will + set a CPU limit on virt-launcher for VMIs running inside namespaces + that match the label selector. The CPU limit will equal the + number of requested vCPUs. This setting does not apply to VMIs + with dedicated CPUs. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object controllerConfiguration: description: ReloadableComponentConfiguration holds all generic k8s configuration options which can be reloaded by components @@ -317,7 +406,87 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + ksmConfiguration: + description: KSMConfiguration holds the information regarding + the enabling the KSM in the nodes (if available). + properties: + nodeLabelSelector: + description: NodeLabelSelector is a selector that filters + in which nodes the KSM will be enabled. Empty NodeLabelSelector + will enable ksm for every node. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + type: object + liveUpdateConfiguration: + description: LiveUpdateConfiguration holds defaults for live update + features + properties: + maxCpuSockets: + description: MaxCpuSockets holds the maximum amount of sockets + that can be hotplugged + format: int32 + type: integer + maxGuest: + anyOf: + - type: integer + - type: string + description: MaxGuest defines the maximum amount memory that + can be allocated to the guest using hotplug. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + maxHotplugRatio: + description: 'MaxHotplugRatio is the ratio used to define + the max amount of a hotplug resource that can be made available + to a VM when the specific Max* setting is not defined (MaxCpuSockets, + MaxGuest) Example: VM is configured with 512Mi of guest + memory, if MaxGuest is not defined and MaxHotplugRatio is + 2 then MaxGuest = 1Gi defaults to 4' + format: int32 + type: integer + type: object machineType: + description: Deprecated. Use architectureConfiguration instead. type: string mediatedDevicesConfiguration: description: MediatedDevicesConfiguration holds information about @@ -337,7 +506,7 @@ spec: nodeMediatedDeviceTypes: items: description: NodeMediatedDeviceTypesConfig holds information - about MDEV types to be defined in a specifc node that + about MDEV types to be defined in a specific node that matches the NodeSelector field. properties: mediatedDeviceTypes: @@ -392,8 +561,8 @@ spec: - type: integer - type: string description: BandwidthPerMigration limits the amount of network - bandwith live migrations are allowed to use. The value is - in quantity per second. Defaults to 0 (no limit) + bandwidth live migrations are allowed to use. The value + is in quantity per second. Defaults to 0 (no limit) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true completionTimeoutPerGiB: @@ -409,6 +578,16 @@ spec: additional layer of live migration encryption provided by KubeVirt. This is usually a bad idea. Defaults to false type: boolean + matchSELinuxLevelOnMigration: + description: By default, the SELinux level of target virt-launcher + pods is forced to the level of the source virt-launcher. + When set to true, MatchSELinuxLevelOnMigration lets the + CRI auto-assign a random level to the target. That will + ensure the target virt-launcher doesn't share categories + with another pod on the node. However, migrations will fail + when using RWX volumes that don't automatically deal with + SELinux levels. + type: boolean network: description: Network is the name of the CNI network to use for live migrations. By default, migrations go through the @@ -517,10 +696,7 @@ spec: resourceName: description: The name of the resource that is representing the device. Exposed by a device plugin and requested - by VMs. Typically of the form vendor.com/product_nameThe - name of the resource that is representing the device. - Exposed by a device plugin and requested by VMs. Typically - of the form vendor.com/product_name + by VMs. Typically of the form vendor.com/product_name type: string required: - pciVendorSelector @@ -528,6 +704,36 @@ spec: type: object type: array x-kubernetes-list-type: atomic + usb: + items: + properties: + externalResourceProvider: + description: If true, KubeVirt will leave the allocation + and monitoring to an external device plugin + type: boolean + resourceName: + description: 'Identifies the list of USB host devices. + e.g: kubevirt.io/storage, kubevirt.io/bootable-usb, + etc' + type: string + selectors: + items: + properties: + product: + type: string + vendor: + type: string + required: + - product + - vendor + type: object + type: array + x-kubernetes-list-type: atomic + required: + - resourceName + type: object + type: array + x-kubernetes-list-type: atomic type: object seccompConfiguration: description: SeccompConfiguration holds Seccomp configuration @@ -578,6 +784,29 @@ spec: description: ResourceRequirements describes the compute resource requirements. properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where + this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -641,6 +870,27 @@ spec: type: object virtualMachineInstancesPerNode: type: integer + virtualMachineOptions: + description: VirtualMachineOptions holds the cluster level information + regarding the virtual machine. + properties: + disableFreePageReporting: + description: DisableFreePageReporting disable the free page + reporting of memory balloon device https://libvirt.org/formatdomain.html#memory-balloon-device. + This will have effect only if AutoattachMemBalloon is not + false and the vmi is not requesting any high performance + feature (dedicatedCPU/realtime/hugePages), in which free + page reporting is always disabled. + type: object + disableSerialConsoleLog: + description: DisableSerialConsoleLog disables logging the + auto-attached default serial console. If not set, serial + console logs will be written to a file and then streamed + from a container named 'guest-console-log'. The value can + be individually overridden for each VM, not relevant if + AutoattachSerialConsole is disabled. + type: object + type: object vmStateStorageClass: description: VMStateStorageClass is the name of the storage class to use for the PVCs created to preserve VM state, like TPM. @@ -1067,9 +1317,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -1129,7 +1376,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -1238,9 +1485,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -1297,7 +1542,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -1408,9 +1653,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -1470,7 +1712,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -1579,9 +1821,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -1638,7 +1878,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -2103,9 +2343,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -2165,7 +2402,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -2274,9 +2511,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -2333,7 +2568,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -2444,9 +2679,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -2506,7 +2738,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -2615,9 +2847,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -2674,7 +2904,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -2788,6 +3018,8 @@ spec: - type type: object type: array + defaultArchitecture: + type: string generations: items: description: GenerationStatus keeps track of the generation for @@ -2856,7 +3088,7 @@ spec: - spec type: object served: true - storage: false + storage: true subresources: status: {} - additionalPrinterColumns: @@ -2866,6 +3098,9 @@ spec: - jsonPath: .status.phase name: Phase type: string + deprecated: true + deprecationWarning: kubevirt.io/v1alpha3 is now deprecated and will be removed + in a future release. name: v1alpha3 schema: openAPIV3Schema: @@ -2972,6 +3207,95 @@ spec: type: object type: object type: object + architectureConfiguration: + properties: + amd64: + properties: + emulatedMachines: + items: + type: string + type: array + x-kubernetes-list-type: atomic + machineType: + type: string + ovmfPath: + type: string + type: object + arm64: + properties: + emulatedMachines: + items: + type: string + type: array + x-kubernetes-list-type: atomic + machineType: + type: string + ovmfPath: + type: string + type: object + defaultArchitecture: + type: string + ppc64le: + properties: + emulatedMachines: + items: + type: string + type: array + x-kubernetes-list-type: atomic + machineType: + type: string + ovmfPath: + type: string + type: object + type: object + autoCPULimitNamespaceLabelSelector: + description: When set, AutoCPULimitNamespaceLabelSelector will + set a CPU limit on virt-launcher for VMIs running inside namespaces + that match the label selector. The CPU limit will equal the + number of requested vCPUs. This setting does not apply to VMIs + with dedicated CPUs. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If + the operator is In or NotIn, the values array must + be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced + during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A + single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is "key", + the operator is "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object controllerConfiguration: description: ReloadableComponentConfiguration holds all generic k8s configuration options which can be reloaded by components @@ -3150,7 +3474,87 @@ spec: description: PullPolicy describes a policy for if/when to pull a container image type: string + ksmConfiguration: + description: KSMConfiguration holds the information regarding + the enabling the KSM in the nodes (if available). + properties: + nodeLabelSelector: + description: NodeLabelSelector is a selector that filters + in which nodes the KSM will be enabled. Empty NodeLabelSelector + will enable ksm for every node. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. + This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + type: object + liveUpdateConfiguration: + description: LiveUpdateConfiguration holds defaults for live update + features + properties: + maxCpuSockets: + description: MaxCpuSockets holds the maximum amount of sockets + that can be hotplugged + format: int32 + type: integer + maxGuest: + anyOf: + - type: integer + - type: string + description: MaxGuest defines the maximum amount memory that + can be allocated to the guest using hotplug. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + maxHotplugRatio: + description: 'MaxHotplugRatio is the ratio used to define + the max amount of a hotplug resource that can be made available + to a VM when the specific Max* setting is not defined (MaxCpuSockets, + MaxGuest) Example: VM is configured with 512Mi of guest + memory, if MaxGuest is not defined and MaxHotplugRatio is + 2 then MaxGuest = 1Gi defaults to 4' + format: int32 + type: integer + type: object machineType: + description: Deprecated. Use architectureConfiguration instead. type: string mediatedDevicesConfiguration: description: MediatedDevicesConfiguration holds information about @@ -3170,7 +3574,7 @@ spec: nodeMediatedDeviceTypes: items: description: NodeMediatedDeviceTypesConfig holds information - about MDEV types to be defined in a specifc node that + about MDEV types to be defined in a specific node that matches the NodeSelector field. properties: mediatedDeviceTypes: @@ -3225,8 +3629,8 @@ spec: - type: integer - type: string description: BandwidthPerMigration limits the amount of network - bandwith live migrations are allowed to use. The value is - in quantity per second. Defaults to 0 (no limit) + bandwidth live migrations are allowed to use. The value + is in quantity per second. Defaults to 0 (no limit) pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ x-kubernetes-int-or-string: true completionTimeoutPerGiB: @@ -3242,6 +3646,16 @@ spec: additional layer of live migration encryption provided by KubeVirt. This is usually a bad idea. Defaults to false type: boolean + matchSELinuxLevelOnMigration: + description: By default, the SELinux level of target virt-launcher + pods is forced to the level of the source virt-launcher. + When set to true, MatchSELinuxLevelOnMigration lets the + CRI auto-assign a random level to the target. That will + ensure the target virt-launcher doesn't share categories + with another pod on the node. However, migrations will fail + when using RWX volumes that don't automatically deal with + SELinux levels. + type: boolean network: description: Network is the name of the CNI network to use for live migrations. By default, migrations go through the @@ -3350,10 +3764,7 @@ spec: resourceName: description: The name of the resource that is representing the device. Exposed by a device plugin and requested - by VMs. Typically of the form vendor.com/product_nameThe - name of the resource that is representing the device. - Exposed by a device plugin and requested by VMs. Typically - of the form vendor.com/product_name + by VMs. Typically of the form vendor.com/product_name type: string required: - pciVendorSelector @@ -3361,6 +3772,36 @@ spec: type: object type: array x-kubernetes-list-type: atomic + usb: + items: + properties: + externalResourceProvider: + description: If true, KubeVirt will leave the allocation + and monitoring to an external device plugin + type: boolean + resourceName: + description: 'Identifies the list of USB host devices. + e.g: kubevirt.io/storage, kubevirt.io/bootable-usb, + etc' + type: string + selectors: + items: + properties: + product: + type: string + vendor: + type: string + required: + - product + - vendor + type: object + type: array + x-kubernetes-list-type: atomic + required: + - resourceName + type: object + type: array + x-kubernetes-list-type: atomic type: object seccompConfiguration: description: SeccompConfiguration holds Seccomp configuration @@ -3411,6 +3852,29 @@ spec: description: ResourceRequirements describes the compute resource requirements. properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in + PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where + this field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map limits: additionalProperties: anyOf: @@ -3474,6 +3938,27 @@ spec: type: object virtualMachineInstancesPerNode: type: integer + virtualMachineOptions: + description: VirtualMachineOptions holds the cluster level information + regarding the virtual machine. + properties: + disableFreePageReporting: + description: DisableFreePageReporting disable the free page + reporting of memory balloon device https://libvirt.org/formatdomain.html#memory-balloon-device. + This will have effect only if AutoattachMemBalloon is not + false and the vmi is not requesting any high performance + feature (dedicatedCPU/realtime/hugePages), in which free + page reporting is always disabled. + type: object + disableSerialConsoleLog: + description: DisableSerialConsoleLog disables logging the + auto-attached default serial console. If not set, serial + console logs will be written to a file and then streamed + from a container named 'guest-console-log'. The value can + be individually overridden for each VM, not relevant if + AutoattachSerialConsole is disabled. + type: object + type: object vmStateStorageClass: description: VMStateStorageClass is the name of the storage class to use for the PVCs created to preserve VM state, like TPM. @@ -3900,9 +4385,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -3962,7 +4444,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -4071,9 +4553,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -4130,7 +4610,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -4241,9 +4721,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -4303,7 +4780,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -4412,9 +4889,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -4471,7 +4946,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -4936,9 +5411,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -4998,7 +5470,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -5107,9 +5579,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -5166,7 +5636,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -5277,9 +5747,6 @@ spec: null selector and null or empty namespaces list means "this pod's namespace". An empty selector ({}) matches all namespaces. - This field is beta-level and is only honored - when PodAffinityNamespaceSelector feature - is enabled. properties: matchExpressions: description: matchExpressions is a list @@ -5339,7 +5806,7 @@ spec: union of the namespaces listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null - namespaceSelector means "this pod's namespace" + namespaceSelector means "this pod's namespace". items: type: string type: array @@ -5448,9 +5915,7 @@ spec: this field and the ones listed in the namespaces field. null selector and null or empty namespaces list means "this pod's namespace". An empty - selector ({}) matches all namespaces. This - field is beta-level and is only honored when - PodAffinityNamespaceSelector feature is enabled. + selector ({}) matches all namespaces. properties: matchExpressions: description: matchExpressions is a list @@ -5507,7 +5972,7 @@ spec: listed in this field and the ones selected by namespaceSelector. null or empty namespaces list and null namespaceSelector means "this - pod's namespace" + pod's namespace". items: type: string type: array @@ -5621,6 +6086,8 @@ spec: - type type: object type: array + defaultArchitecture: + type: string generations: items: description: GenerationStatus keeps track of the generation for @@ -5689,7 +6156,7 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} @@ -5788,6 +6255,18 @@ rules: - routes/custom-host verbs: - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - delete + - update + - create + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -5983,22 +6462,6 @@ rules: - list - watch - patch -- apiGroups: - - flavor.kubevirt.io - resources: - - virtualmachineflavors - - virtualmachineclusterflavors - - virtualmachinepreferences - - virtualmachineclusterpreferences - verbs: - - get - - delete - - create - - update - - patch - - list - - watch - - deletecollection - apiGroups: - "" resources: @@ -6098,6 +6561,14 @@ rules: - get - list - watch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - watch - apiGroups: - instancetype.kubevirt.io resources: @@ -6280,7 +6751,8 @@ rules: - virtualmachineinstances/freeze - virtualmachineinstances/unfreeze - virtualmachineinstances/softreboot - - virtualmachineinstances/addinterface + - virtualmachineinstances/sev/setupsession + - virtualmachineinstances/sev/injectlaunchsecret verbs: - update - apiGroups: @@ -6295,8 +6767,6 @@ rules: - network-attachment-definitions verbs: - get - - list - - watch - apiGroups: - apiextensions.k8s.io resources: @@ -6376,6 +6846,13 @@ rules: - namespaces verbs: - get +- apiGroups: + - "" + resources: + - resourcequotas + verbs: + - list + - watch - apiGroups: - route.openshift.io resources: @@ -6400,6 +6877,18 @@ rules: - list - get - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - delete + - update + - create + - patch - apiGroups: - kubevirt.io resources: @@ -6507,6 +6996,8 @@ rules: - virtualmachineinstances/guestosinfo - virtualmachineinstances/filesystemlist - virtualmachineinstances/userlist + - virtualmachineinstances/sev/fetchcertchain + - virtualmachineinstances/sev/querylaunchmeasurement verbs: - get - apiGroups: @@ -6519,7 +7010,8 @@ rules: - virtualmachineinstances/freeze - virtualmachineinstances/unfreeze - virtualmachineinstances/softreboot - - virtualmachineinstances/addinterface + - virtualmachineinstances/sev/setupsession + - virtualmachineinstances/sev/injectlaunchsecret verbs: - update - apiGroups: @@ -6539,7 +7031,6 @@ rules: - virtualmachines/removevolume - virtualmachines/migrate - virtualmachines/memorydump - - virtualmachines/addinterface verbs: - update - apiGroups: @@ -6653,6 +7144,8 @@ rules: - virtualmachineinstances/guestosinfo - virtualmachineinstances/filesystemlist - virtualmachineinstances/userlist + - virtualmachineinstances/sev/fetchcertchain + - virtualmachineinstances/sev/querylaunchmeasurement verbs: - get - apiGroups: @@ -6665,7 +7158,8 @@ rules: - virtualmachineinstances/freeze - virtualmachineinstances/unfreeze - virtualmachineinstances/softreboot - - virtualmachineinstances/addinterface + - virtualmachineinstances/sev/setupsession + - virtualmachineinstances/sev/injectlaunchsecret verbs: - update - apiGroups: @@ -6685,7 +7179,6 @@ rules: - virtualmachines/removevolume - virtualmachines/migrate - virtualmachines/memorydump - - virtualmachines/addinterface verbs: - update - apiGroups: @@ -6797,6 +7290,8 @@ rules: - virtualmachineinstances/guestosinfo - virtualmachineinstances/filesystemlist - virtualmachineinstances/userlist + - virtualmachineinstances/sev/fetchcertchain + - virtualmachineinstances/sev/querylaunchmeasurement verbs: - get - apiGroups: @@ -6870,6 +7365,15 @@ rules: - get - list - watch +- apiGroups: + - instancetype.kubevirt.io + resources: + - virtualmachineclusterinstancetypes + - virtualmachineclusterpreferences + verbs: + - get + - list + - watch - apiGroups: - authentication.k8s.io resources: @@ -6943,30 +7447,30 @@ spec: - virt-operator env: - name: VIRT_OPERATOR_IMAGE - value: index.docker.io/naimingshen/virt-operator@sha256:2eb976700078429ba25df55dcd923c1b7764eb4c60762094c71b4796ffb45f9c + value: index.docker.io/naimingshen/virt-operator@sha256:c6649283b793913d64c19cd31e6b2f17c1f9cc8c8846f18352b4b5b2e90f04be - name: WATCH_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] - name: VIRT_API_SHASUM - value: sha256:f4ee0fc58a229f4febd4b5e9fe2a0821aa6cff048aea9532a00019c43af44ffb + value: sha256:d033bf2425c8be556e774467bcce5fff431a4d889c688ba9a92bef2a33ab540e - name: VIRT_CONTROLLER_SHASUM - value: sha256:80df9ada98043e2f304059c90c27649d2fec95c77d42a3e5b87b23c276264d2d + value: sha256:fd9d8a5f21f8c4d64f9c4e66b5aa3f8fb7a3c10d969649dad8b2bc8245de2f86 - name: VIRT_HANDLER_SHASUM - value: sha256:2d88e7359886842d6c84c6e8f1dc8849e26f48ffe1b6ccccae2f13835ee37ae4 + value: sha256:25dc1e2b6166db62ee28cca955cba0ee86a4870f27cdc407348d8b0ba5cfbbeb - name: VIRT_LAUNCHER_SHASUM - value: sha256:f8a53eeaaee43a899385c56ca36970e293712323f8e4fab8e4e71c93f6c27993 + value: sha256:66c37c3019b43ddc502dfa02d989b6c90a6974308cb9e87e8ff27280f592fc15 - name: VIRT_EXPORTPROXY_SHASUM - value: sha256:3724cb4f953f4c96d40ae524a7330ce0cd418ebec919f0cee78f6574f8ea3a37 + value: sha256:1769c4be68d9d5b34eb5cd59ec39f05acc20bacc28fc84fb572065c51c2046e1 - name: VIRT_EXPORTSERVER_SHASUM - value: sha256:369caa036568330294bcd9b99e4a865cf878c96d2f95cbd14a976c46479349d7 + value: sha256:1c520f67ab02bae9c412817ef147eaba94869a6fb45ca06def04c1538dc26f43 - name: GS_SHASUM - value: sha256:d6e13c109eb4dab5adebc6eda16bb075440346d99238bafb2d0c07d0dfaab0ab + value: sha256:49fc603723d479ba145c001d1feea3fc803a66e031bfa9947b3b56f534855a71 - name: PR_HELPER_SHASUM - value: sha256:080d7d4763918091d59d059a09a9d140eb6cf24a2e57f1357130c6e102df24ab + value: sha256:24368e553dc7bc52089a734196e7634e306cab33dc49c205c93ccb3162e5c0b9 - name: KUBEVIRT_VERSION - value: latest2 - image: index.docker.io/naimingshen/virt-operator@sha256:2eb976700078429ba25df55dcd923c1b7764eb4c60762094c71b4796ffb45f9c + value: latest + image: index.docker.io/naimingshen/virt-operator@sha256:c6649283b793913d64c19cd31e6b2f17c1f9cc8c8846f18352b4b5b2e90f04be imagePullPolicy: IfNotPresent name: virt-operator ports: diff --git a/pkg/kube/longhorn-generate-support-bundle.sh b/pkg/kube/longhorn-generate-support-bundle.sh new file mode 100755 index 0000000000..5bef360549 --- /dev/null +++ b/pkg/kube/longhorn-generate-support-bundle.sh @@ -0,0 +1,44 @@ +#!/bin/sh + +LOG_DIR=/persist/newlog/kube + +echo "Apply longhorn support bundle yaml at $(date)" + +cat < "${LOG_DIR}/longhorn${filename}" +echo "longhorn support bundle downloaded to ${LOG_DIR}/longhorn${filename}" + +kubectl -n longhorn-system delete supportbundle.longhorn.io/support-bundle-collect-info diff --git a/pkg/kube/nsmounter b/pkg/kube/nsmounter new file mode 100755 index 0000000000..7e623e4515 --- /dev/null +++ b/pkg/kube/nsmounter @@ -0,0 +1,46 @@ +#!/bin/bash + +PROC_DIR="/proc" + +os_distro_eve="eve" +os_distro="" + +get_os_distro() { + local -r version_info=$(< $PROC_DIR/version) + + [[ $version_info =~ $os_distro_eve ]] && os_distro=$os_distro_eve +} + +target_pid=1 + +get_pid() { + local process_name=$1 + local pid + local status_file + local name + + for dir in "$PROC_DIR"/*/; do + pid=$(basename "$dir") + status_file="$PROC_DIR/$pid/status" + + if [ -f "$status_file" ]; then + while IFS= read -r line; do + if [[ $line == "Name:"* ]]; then + name="${line#*:}" + name="${name//[$'\t ']/}" # Remove both spaces and tabs + break # Exit the loop once the Name is found + fi + done < "$status_file" + fi + + if [ "$name" = "$process_name" ]; then + target_pid=$pid + fi + done +} + +get_os_distro + +[[ $os_distro = "$os_distro_eve" ]] && get_pid "cluster-init.sh" + +nsenter -t "$target_pid" -m -n -u -- "$@"