🌱 use ubuntu-22.04 (#1201)

🌱 Update from Ubuntu 20.04 to 22.04

Tiltfile

@@ -142,9 +142,7 @@ def caph():
             yaml = str(encode_yaml_stream(yaml_dict))
             yaml = fixup_yaml_empty_arrays(yaml)
 
-    # copy things from data directory to .tiltbuild 
-    if not os.path.exists('.tiltbuild/installimage.tgz'):
-        local("cp data/hetzner-installimage-v1.0.5.tgz .tiltbuild/installimage.tgz")
+    local("cp data/hetzner-installimage-v1.0.5.tgz .tiltbuild/installimage.tgz")
 
     # Set up a local_resource build of the provider's manager binary.
 

api/v1beta1/hetznerbaremetalmachine_types_test.go

@@ -277,9 +277,9 @@ func Test_Image_String(t *testing.T) {
 		{
 			Image{
 				Name: "nfs",
-				Path: "/root/.oldroot/nfs/install/../images/Ubuntu-2004-focal-64-minimal-hwe.tar.gz",
+				Path: "/root/.oldroot/nfs/images/Ubuntu-2204-jammy-amd64-base.tar.gz",
 			},
-			"nfs (/root/.oldroot/nfs/install/../images/Ubuntu-2004-focal-64-minimal-hwe.tar.gz)",
+			"nfs (/root/.oldroot/nfs/images/Ubuntu-2204-jammy-amd64-base.tar.gz)",
 		},
 	} {
 		require.Equal(t, row.expected, row.image.String())

controllers/hetznerbaremetalhost_controller_test.go

@@ -884,6 +884,11 @@ name="eth0" model="Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express
 		StdErr: "",
 		Err:    nil,
 	})
+	sshClient.On("GetHardwareDetailsDebug").Return(sshclient.Output{
+		StdOut: "Dummy outupt",
+		StdErr: "",
+		Err:    nil,
+	})
 	sshClient.On("DownloadImage", mock.Anything, mock.Anything).Return(sshclient.Output{})
 	sshClient.On("CreateAutoSetup", mock.Anything).Return(sshclient.Output{})
 	sshClient.On("UntarTGZ").Return(sshclient.Output{})
@@ -892,4 +897,5 @@ name="eth0" model="Realtek Semiconductor Co., Ltd. RTL8111/8168/8411 PCI Express
 	sshClient.On("Reboot").Return(sshclient.Output{})
 	sshClient.On("GetCloudInitOutput").Return(sshclient.Output{StdOut: "dummy content of /var/log/cloud-init-output.log"})
 	sshClient.On("DetectLinuxOnAnotherDisk", mock.Anything).Return(sshclient.Output{})
+	sshClient.On("GetRunningInstallImageProcesses").Return(sshclient.Output{})
 }

pkg/services/baremetal/client/ssh/ssh_client.go

@@ -188,6 +188,8 @@ type Client interface {
 	GetHardwareDetailsCPUFlags() Output
 	GetHardwareDetailsCPUThreads() Output
 	GetHardwareDetailsCPUCores() Output
+	GetHardwareDetailsDebug() Output
+	GetRunningInstallImageProcesses() Output
 	GetCloudInitOutput() Output
 	CreateAutoSetup(data string) Output
 	DownloadImage(path, url string) Output
@@ -250,7 +252,7 @@ func (c *sshClient) GetHardwareDetailsRAM() Output {
 
 // GetHardwareDetailsNics implements the GetHardwareDetailsNics method of the SSHClient interface.
 func (c *sshClient) GetHardwareDetailsNics() Output {
-	out := c.runSSH(`cat << 'EOF' > nic-info.sh
+	out := c.runSSH(`cat << 'EOF_VIA_SSH' > nic-info.sh
 #!/bin/sh
 for iname in $(ip a |awk '/state UP/{print $2}' | sed 's/://')
 do
@@ -270,7 +272,7 @@ if test -n $IP_V6; then
 fi
 
 done
-EOF`)
+EOF_VIA_SSH`)
 	if out.Err != nil || out.StdErr != "" {
 		return out
 	}
@@ -322,11 +324,19 @@ func (c *sshClient) GetHardwareDetailsCPUCores() Output {
 	return c.runSSH(`grep 'cpu cores' /proc/cpuinfo | uniq | awk '{print $4}'`)
 }
 
+// GetHardwareDetailsDebug implements the GetHardwareDetailsDebug method of the SSHClient interface.
+func (c *sshClient) GetHardwareDetailsDebug() Output {
+	return c.runSSH(`ip a; echo ==========----------==========; 
+	ethtool "*"; echo ==========----------==========;
+	lspci; echo ==========----------==========;
+	`)
+}
+
 // CreateAutoSetup implements the CreateAutoSetup method of the SSHClient interface.
 func (c *sshClient) CreateAutoSetup(data string) Output {
-	return c.runSSH(fmt.Sprintf(`cat << 'EOF' > /autosetup
+	return c.runSSH(fmt.Sprintf(`cat << 'EOF_VIA_SSH' > /autosetup
 %s
-EOF`, data))
+EOF_VIA_SSH`, data))
 }
 
 // DownloadImage implements the DownloadImage method of the SSHClient interface.
@@ -345,15 +355,21 @@ OCI_REGISTRY_AUTH_TOKEN=%s /root/download-from-oci.sh %s %s`, downloadFromOciShe
 
 // CreatePostInstallScript implements the CreatePostInstallScript method of the SSHClient interface.
 func (c *sshClient) CreatePostInstallScript(data string) Output {
-	out := c.runSSH(fmt.Sprintf(`cat << 'EOF' > /root/post-install.sh
-	%sEOF`, data))
+	out := c.runSSH(fmt.Sprintf(`cat << 'EOF_VIA_SSH' > /root/post-install.sh
+%s
+EOF_VIA_SSH`, data))
 
 	if out.Err != nil || out.StdErr != "" {
 		return out
 	}
 	return c.runSSH(`chmod +x /root/post-install.sh . `)
 }
 
+// GetRunningInstallImageProcesses returns the running installimage processes. Output.StdOut is empty if no processes are running.
+func (c *sshClient) GetRunningInstallImageProcesses() Output {
+	return c.runSSH(`ps aux| grep installimage | grep -v grep; true`)
+}
+
 // ExecuteInstallImage implements the ExecuteInstallImage method of the SSHClient interface.
 func (c *sshClient) ExecuteInstallImage(hasPostInstallScript bool) Output {
 	var cmd string
@@ -363,13 +379,13 @@ func (c *sshClient) ExecuteInstallImage(hasPostInstallScript bool) Output {
 		cmd = `/root/hetzner-installimage/installimage -a -c /autosetup`
 	}
 
-	out := c.runSSH(fmt.Sprintf(`cat << 'EOF' > /root/installimage-wrapper.sh
+	out := c.runSSH(fmt.Sprintf(`cat << 'EOF_VIA_SSH' > /root/installimage-wrapper.sh
 #!/bin/bash
 export TERM=xterm
 
 # don't wait 20 seconds before starting: echo "x"
 echo "x" | %s
-EOF`, cmd))
+EOF_VIA_SSH`, cmd))
 	if out.Err != nil || out.StdErr != "" {
 		return out
 	}
@@ -428,15 +444,16 @@ func (c *sshClient) CreateNoCloudDirectory() Output {
 
 // CreateMetaData implements the CreateMetaData method of the SSHClient interface.
 func (c *sshClient) CreateMetaData(hostName string) Output {
-	return c.runSSH(fmt.Sprintf(`cat << 'EOF' > /var/lib/cloud/seed/nocloud-net/meta-data
+	return c.runSSH(fmt.Sprintf(`cat << 'EOF_VIA_SSH' > /var/lib/cloud/seed/nocloud-net/meta-data
 local-hostname: %s
-EOF`, hostName))
+EOF_VIA_SSH`, hostName))
 }
 
 // CreateUserData implements the CreateUserData method of the SSHClient interface.
 func (c *sshClient) CreateUserData(userData string) Output {
-	return c.runSSH(fmt.Sprintf(`cat << 'EOF' > /var/lib/cloud/seed/nocloud-net/user-data
-%sEOF`, userData))
+	return c.runSSH(fmt.Sprintf(`cat << 'EOF_VIA_SSH' > /var/lib/cloud/seed/nocloud-net/user-data
+%s
+EOF_VIA_SSH`, userData))
 }
 
 // CloudInitStatus implements the CloudInitStatus method of the SSHClient interface.
@@ -488,9 +505,9 @@ func (c *sshClient) ResetKubeadm() Output {
 }
 
 func (c *sshClient) DetectLinuxOnAnotherDisk(sliceOfWwns []string) Output {
-	return c.runSSH(fmt.Sprintf(`cat <<'EOF' | bash -s -- %s
+	return c.runSSH(fmt.Sprintf(`cat <<'EOF_VIA_SSH' | bash -s -- %s
 %s
-EOF
+EOF_VIA_SSH
 `, strings.Join(sliceOfWwns, " "), detectLinuxOnAnotherDiskShellScript))
 }
 

pkg/services/baremetal/host/host.go

@@ -537,6 +537,12 @@ func (s *Service) actionRegistering() actionResult {
 		return actionContinue{delay: 10 * time.Second}
 	}
 
+	output := sshClient.GetHardwareDetailsDebug()
+	if output.Err != nil {
+		return actionError{err: fmt.Errorf("failed to obtain hardware for debugging: %w", output.Err)}
+	}
+	record.Eventf(s.scope.HetznerBareMetalHost, "GetHardwareDetails", "%s\n\nstderr:\n%s", output.StdOut, out.StdErr)
+
 	if s.scope.HetznerBareMetalHost.Spec.Status.HardwareDetails == nil {
 		hardwareDetails, err := getHardwareDetails(sshClient)
 		if err != nil {
@@ -928,7 +934,7 @@ func getCPUFlags(sshClient sshclient.Client) ([]string, error) {
 
 func handleSSHError(out sshclient.Output) error {
 	if out.Err != nil {
-		return fmt.Errorf("failed to perform ssh command: %w", out.Err)
+		return fmt.Errorf("failed to perform ssh command: stdout %q. stderr %q. %w", out.StdOut, out.StdErr, out.Err)
 	}
 	if out.StdErr != "" {
 		return fmt.Errorf("%w: StdErr: %s", errSSHStderr, out.StdErr)
@@ -987,7 +993,7 @@ func (s *Service) actionImageInstalling() actionResult {
 
 		// Some other error like connection timeout. Retry again later.
 		// This often during provisioning.
-		msg := fmt.Sprintf("DetectLinuxOnAnotherDisk failed (will retry): %s. StdErr: %s (%s)",
+		msg := fmt.Sprintf("will retry: %s. StdErr: %s (%s)",
 			out.StdOut, out.StdErr, out.Err.Error())
 		conditions.MarkFalse(
 			s.scope.HetznerBareMetalHost,
@@ -1002,6 +1008,20 @@ func (s *Service) actionImageInstalling() actionResult {
 		}
 	}
 
+	// if the previous reconcile was stopped, then wait until the first
+	// run of installimage was finished.
+	out = sshClient.GetRunningInstallImageProcesses()
+	if out.Err != nil {
+		return actionError{err: fmt.Errorf("failed to get running installimage processes: %q %q %w", out.StdOut, out.StdErr, out.Err)}
+	}
+	if out.StdOut != "" {
+		record.Warnf(s.scope.HetznerBareMetalHost, "InstallImageAlreadyRunning",
+			"installimage is already running:\n%s", out.StdOut)
+		return actionContinue{
+			delay: 10 * time.Second,
+		}
+	}
+
 	record.Event(s.scope.HetznerBareMetalHost, "InstallImagePreflightCheckSuccessful", "Rescue system reachable, disks look good.")
 
 	autoSetupInput, actionRes := s.createAutoSetupInput(sshClient)

pkg/services/baremetal/host/host_test.go

@@ -1103,7 +1103,7 @@ var _ = Describe("actionRegistering", func() {
 			sshMock.On("GetHardwareDetailsCPUFlags").Return(sshclient.Output{StdOut: "flag1 flag2 flag3"})
 			sshMock.On("GetHardwareDetailsCPUThreads").Return(sshclient.Output{StdOut: "123"})
 			sshMock.On("GetHardwareDetailsCPUCores").Return(sshclient.Output{StdOut: "12"})
-
+			sshMock.On("GetHardwareDetailsDebug").Return(sshclient.Output{StdOut: "Dummy outupt"})
 			service := newTestService(host, nil, bmmock.NewSSHFactory(sshMock, sshMock, sshMock), nil, helpers.GetDefaultSSHSecret(rescueSSHKeyName, "default"))
 
 			actResult := service.actionRegistering()

pkg/services/hcloud/server/server_suite_test.go

@@ -79,14 +79,14 @@ const serverJSON = `
 		},
 		"deleted": null,
 		"deprecated": "2018-02-28T00:00:00+00:00",
-		"description": "Ubuntu 20.04 Standard 64 bit",
+		"description": "Ubuntu 22.04 Standard 64 bit",
 		"disk_size": 10,
 		"id": 42,
 		"image_size": 2.3,
 		"labels": {},
-		"name": "ubuntu-20.04",
+		"name": "ubuntu-22.04",
 		"os_flavor": "ubuntu",
-		"os_version": "20.04",
+		"os_version": "22.04",
 		"protection": {
 			"delete": false
 		},

templates/cluster-templates/bases/hcloud-kcp-fedora.yaml

@@ -251,8 +251,8 @@ spec:
       - dnf install --setopt=obsoletes=0 -y kubelet-0:1.23.4-0 kubeadm-0:1.23.4-0 kubectl-0:1.23.4-0 python3-dnf-plugin-versionlock bash-completion --disableexcludes=kubernetes && dnf versionlock kubelet kubectl kubeadm && systemctl enable kubelet && systemctl start crio && kubeadm config images pull --kubernetes-version 1.23.4
       - dnf install -y policycoreutils-python-utils
       - semanage fcontext -a -t container_file_t /var/lib/etcd && mkdir -p /var/lib/etcd && restorecon -rv /var /etc
-      - echo 'source <(kubectl completion bash)' >>~/.bashrc
-      - echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >>~/.bashrc
+      - echo 'source <(kubectl completion bash)' >>/root/.bashrc
+      - echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >>/root/.bashrc
       - setenforce 0 && sed -i -e '/^\(#\|\)SELINUX/s/^.*$/SELINUX=disabled/' /etc/selinux/config
       - dnf -y remove firewalld
       - dnf -y autoremove && dnf -y clean all

templates/cluster-templates/bases/hcloud-kcp-ubuntu.yaml

@@ -194,7 +194,7 @@ spec:
       - apt-get update
       - apt-get install -y kubelet=$KUBERNETES_VERSION-1.1 kubeadm=$KUBERNETES_VERSION-1.1 kubectl=$KUBERNETES_VERSION-1.1  bash-completion && apt-mark hold kubelet kubectl kubeadm && systemctl enable kubelet
       - kubeadm config images pull --kubernetes-version $KUBERNETES_VERSION
-      - echo 'source <(kubectl completion bash)' >>~/.bashrc
-      - echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >>~/.bashrc
+      - echo 'source <(kubectl completion bash)' >>/root/.bashrc
+      - echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >>/root/.bashrc
       - apt-get -y autoremove && apt-get -y clean all
   version: "${KUBERNETES_VERSION}"

templates/cluster-templates/bases/hetznerbaremetal-kcp-ubuntu.yaml

@@ -211,7 +211,7 @@ spec:
       - apt-get update
       - apt-get install -y kubelet=$KUBERNETES_VERSION-1.1 kubeadm=$KUBERNETES_VERSION-1.1 kubectl=$KUBERNETES_VERSION-1.1  bash-completion && apt-mark hold kubelet kubectl kubeadm && systemctl enable kubelet
       - kubeadm config images pull --kubernetes-version $KUBERNETES_VERSION
-      - echo 'source <(kubectl completion bash)' >>~/.bashrc
-      - echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >>~/.bashrc
+      - echo 'source <(kubectl completion bash)' >>/root/.bashrc
+      - echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >>/root/.bashrc
       - apt-get -y autoremove && apt-get -y clean all
   version: "${KUBERNETES_VERSION}"

templates/cluster-templates/bases/hetznerbaremetal-mt-control-plane-ubuntu.yaml

@@ -7,7 +7,7 @@ spec:
     spec:
       installImage:
         image:
-          path: /root/.oldroot/nfs/install/../images/Ubuntu-2004-focal-64-minimal-hwe.tar.gz
+          path: /root/.oldroot/nfs/images/Ubuntu-2204-jammy-amd64-base.tar.gz
         partitions:
           - fileSystem: esp
             mount: /boot/efi
@@ -23,6 +23,7 @@ spec:
           mkdir -p /etc/cloud/cloud.cfg.d && touch /etc/cloud/cloud.cfg.d/99-custom-networking.cfg
           echo "network: { config: disabled }" > /etc/cloud/cloud.cfg.d/99-custom-networking.cfg
           apt-get update && apt-get install -y cloud-init apparmor apparmor-utils
+          cloud-init clean --logs
       sshSpec:
         portAfterInstallImage: 22
         portAfterCloudInit: 22