ci: add workaround for issue #515

tests/e2e/backup-restore_test.go

@@ -43,11 +43,8 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 			if backupRestoreVersion != "" {
 				chartRepo = "https://github.com/rancher/backup-restore-operator/releases/download/" + backupRestoreVersion
 			} else {
-				err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", chartRepo, "https://charts.rancher.io")
-				Expect(err).To(Not(HaveOccurred()))
-
-				err = kubectl.RunHelmBinaryWithCustomErr("repo", "update")
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry("repo", "add", chartRepo, "https://charts.rancher.io")
+				RunHelmCmdWithRetry("repo", "update")
 			}
 		})
 
@@ -64,6 +61,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 					"upgrade", "--install", chart, chartRepo + "/" + chartName,
 					"--namespace", "cattle-resources-system",
 					"--create-namespace",
+					"--wait", "--wait-for-jobs",
 				}
 
 				// Add specific options for the rancher-backup chart
@@ -75,8 +73,7 @@ var _ = Describe("E2E - Install Backup/Restore Operator", Label("install-backup-
 				}
 
 				// Install through Helm
-				err := kubectl.RunHelmBinaryWithCustomErr(flags...)
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry(flags...)
 			}
 		})
 

tests/e2e/helpers/elemental/elemental.go

@@ -23,21 +23,50 @@ import (
 )
 
 /**
- * Get MachineInventory name (aka. server id)
- * @remarks The repository is added to the local cluster
+ * Add node selector
+ * @remarks A nodeSelector field is added
+ * @param key key to add in YAML
+ * @param value value to set the key to
+ * @returns The YAML structure or an error
+ */
+func AddSelector(key, value string) ([]byte, error) {
+	type selectorYaml struct {
+		MatchLabels map[string]string `yaml:"matchLabels,omitempty"`
+	}
+
+	type selector struct {
+		SelectorYaml selectorYaml `yaml:"nodeSelector,omitempty"`
+	}
+
+	v := selectorYaml{map[string]string{key: value}}
+	s := selector{v}
+	out, err := yaml.Marshal(s)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add indent at the beginning
+	out = append([]byte("  "), out...)
+
+	return out, nil
+}
+
+/**
+ * Get Machine from MachineInventory
+ * @remarks This is useful to link machine name from Elemental to the Rancher Manager one
  * @param clusterNS Name of the repository
- * @param index URL of the repository
- * @returns The name/id of the server or an error
+ * @param machineInventory Machine name as seen by Elemental
+ * @returns Corresponding internal machine name
  */
-func GetServerID(clusterNS string, index int) (string, error) {
-	serverID, err := kubectl.Run("get", "MachineInventories",
+func GetInternalMachine(clusterNS, machineInventory string) (string, error) {
+	machine, err := kubectl.Run("get", "Machine",
 		"--namespace", clusterNS,
-		"-o", "jsonpath={.items["+fmt.Sprint(index-1)+"].metadata.name}")
+		"-o", "jsonpath={.items[?(@.status.nodeRef.name==\""+machineInventory+"\")].metadata.name}")
 	if err != nil {
 		return "", err
 	}
 
-	return serverID, nil
+	return machine, nil
 }
 
 /**
@@ -74,32 +103,20 @@ func GetOperatorVersion() (string, error) {
 }
 
 /**
- * Add node selector
- * @remarks A nodeSelector field is added
- * @param key key to add in YAML
- * @param value value to set the key to
- * @returns The YAML structure or an error
+ * Get MachineInventory name (aka. server id)
+ * @param clusterNS Name of the repository
+ * @param index URL of the repository
+ * @returns The name/id of the server or an error
  */
-func AddSelector(key, value string) ([]byte, error) {
-	type selectorYaml struct {
-		MatchLabels map[string]string `yaml:"matchLabels,omitempty"`
-	}
-
-	type selector struct {
-		SelectorYaml selectorYaml `yaml:"nodeSelector,omitempty"`
-	}
-
-	v := selectorYaml{map[string]string{key: value}}
-	s := selector{v}
-	out, err := yaml.Marshal(s)
+func GetServerID(clusterNS string, index int) (string, error) {
+	serverID, err := kubectl.Run("get", "MachineInventories",
+		"--namespace", clusterNS,
+		"-o", "jsonpath={.items["+fmt.Sprint(index-1)+"].metadata.name}")
 	if err != nil {
-		return nil, err
+		return "", err
 	}
 
-	// Add indent at the beginning
-	out = append([]byte("  "), out...)
-
-	return out, nil
+	return serverID, nil
 }
 
 /**

tests/e2e/install_test.go

@@ -162,33 +162,30 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
 			})
 		} else {
 			By("Installing CertManager", func() {
-				err := kubectl.RunHelmBinaryWithCustomErr("repo", "add", "jetstack", "https://charts.jetstack.io")
-				Expect(err).To(Not(HaveOccurred()))
-
-				err = kubectl.RunHelmBinaryWithCustomErr("repo", "update")
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry("repo", "add", "jetstack", "https://charts.jetstack.io")
+				RunHelmCmdWithRetry("repo", "update")
 
 				// Set flags for cert-manager installation
 				flags := []string{
 					"upgrade", "--install", "cert-manager", "jetstack/cert-manager",
 					"--namespace", "cert-manager",
 					"--create-namespace",
 					"--set", "installCRDs=true",
+					"--wait", "--wait-for-jobs",
 				}
 
 				if clusterType == "hardened" {
 					flags = append(flags, "--version", CertManagerVersion)
 				}
 
-				err = kubectl.RunHelmBinaryWithCustomErr(flags...)
-				Expect(err).To(Not(HaveOccurred()))
+				RunHelmCmdWithRetry(flags...)
 
 				checkList := [][]string{
 					{"cert-manager", "app.kubernetes.io/component=controller"},
 					{"cert-manager", "app.kubernetes.io/component=webhook"},
 					{"cert-manager", "app.kubernetes.io/component=cainjector"},
 				}
-				err = rancher.CheckPod(k, checkList)
+				err := rancher.CheckPod(k, checkList)
 				Expect(err).To(Not(HaveOccurred()))
 			})
 		}
@@ -306,13 +303,13 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), func() {
 						continue
 					}
 				}
-				Eventually(func() error {
-					return kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart,
-						operatorRepo+"/"+chart+"-chart",
-						"--namespace", "cattle-elemental-system",
-						"--create-namespace",
-					)
-				}, tools.SetTimeout(1*time.Minute), 10*time.Second).Should(BeNil())
+
+				RunHelmCmdWithRetry("upgrade", "--install", chart,
+					operatorRepo+"/"+chart+"-chart",
+					"--namespace", "cattle-elemental-system",
+					"--create-namespace",
+					"--wait", "--wait-for-jobs",
+				)
 			}
 
 			// Wait for pod to be started

tests/e2e/reset_test.go

@@ -15,38 +15,42 @@ limitations under the License.
 package e2e_test
 
 import (
-	"os/exec"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rancher-sandbox/ele-testhelpers/kubectl"
 	"github.com/rancher-sandbox/ele-testhelpers/tools"
+	"github.com/rancher/elemental/tests/e2e/helpers/elemental"
 )
 
 var _ = Describe("E2E - Test the reset feature", Label("reset"), func() {
 	It("Reset one node in the cluster", func() {
 		// Get the machine inventory name list
-		machineInventory, err := kubectl.Run("get", "machineinventory", "-A", "-o", "jsonpath='{.items[*].metadata.name}'")
+		machineInventory, err := kubectl.Run("get", "MachineInventory",
+			"--namespace", clusterNS,
+			"-o", "jsonpath='{.items[*].metadata.name}'")
 		Expect(err).To(Not(HaveOccurred()))
 		firstMachineInventory := strings.Split(machineInventory, " ")[1]
 
-		By("Configuring reset at machine inventory level", func() {
+		By("Configuring reset at MachineInventory level", func() {
 			// Patch the first machine inventory to enable reset
-			_, err = kubectl.Run("patch", "machineinventory", firstMachineInventory, "--namespace", clusterNS, "--type", "merge", "--patch-file", resetMachineInv)
+			_, err = kubectl.Run("patch", "MachineInventory", firstMachineInventory,
+				"--namespace", clusterNS, "--type", "merge",
+				"--patch-file", resetMachineInv)
 			Expect(err).To(Not(HaveOccurred()))
 		})
 
 		By("Deleting and removing the node from the cluster", func() {
-			out, err := exec.Command("bash", "-c", "kubectl get machines -A | awk '/"+firstMachineInventory+"/ {print $2}'").CombinedOutput()
+			machineToRemove, err := elemental.GetInternalMachine(clusterNS, firstMachineInventory)
 			Expect(err).To(Not(HaveOccurred()))
-			machineToRemove := strings.Trim(string(out), "\n")
-			_, err = kubectl.Run("delete", "machines", machineToRemove, "-n", "fleet-default")
+			_, err = kubectl.Run("delete", "machines", machineToRemove,
+				"--namespace", clusterNS)
 			Expect(err).To(Not(HaveOccurred()))
 		})
 
-		By("Checking that machine inventory is deleted", func() {
+		By("Checking that MachineInventory is deleted", func() {
 			Eventually(func() string {
 				out, _ := kubectl.Run("get", "MachineInventory",
 					"--namespace", clusterNS,
@@ -55,7 +59,7 @@ var _ = Describe("E2E - Test the reset feature", Label("reset"), func() {
 			}, tools.SetTimeout(5*time.Minute), 5*time.Second).ShouldNot(ContainSubstring(firstMachineInventory))
 		})
 
-		By("Checking that machine inventory is back after the reset", func() {
+		By("Checking that MachineInventory is back after the reset", func() {
 			Eventually(func() string {
 				out, _ := kubectl.Run("get", "MachineInventory",
 					"--namespace", clusterNS,

tests/e2e/suite_test.go

@@ -178,6 +178,12 @@ func GetNodeInfo(hostName string) (*tools.Client, string) {
 	return c, hostData.Mac
 }
 
+func RunHelmCmdWithRetry(s ...string) {
+	Eventually(func() error {
+		return kubectl.RunHelmBinaryWithCustomErr(s...)
+	}, tools.SetTimeout(2*time.Minute), 20*time.Second).Should(Not(HaveOccurred()))
+}
+
 func FailWithReport(message string, callerSkip ...int) {
 	// Ensures the correct line numbers are reported
 	Fail(message, callerSkip[0]+1)

tests/e2e/uninstall-operator_test.go

@@ -17,15 +17,25 @@ package e2e_test
 import (
 	"os/exec"
 	"strings"
+	"sync"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/rancher-sandbox/ele-testhelpers/kubectl"
 	"github.com/rancher-sandbox/ele-testhelpers/rancher"
 	"github.com/rancher-sandbox/ele-testhelpers/tools"
+	"github.com/rancher/elemental/tests/e2e/helpers/elemental"
 )
 
+func deleteFinalizers(ns, object, value string) {
+	_, err := kubectl.Run("patch", object,
+		"--namespace", ns, value, "--type", "merge",
+		"--patch", "{\"metadata\":{\"finalizers\":null}}")
+	Expect(err).To(Not(HaveOccurred()))
+
+}
+
 func testClusterAvailability(ns, cluster string) {
 	Eventually(func() string {
 		out, _ := kubectl.Run("get", "cluster",
@@ -64,11 +74,10 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator
 						continue
 					}
 				}
-				err := kubectl.RunHelmBinaryWithCustomErr(
+				RunHelmCmdWithRetry(
 					"uninstall", chart,
 					"--namespace", "cattle-elemental-system",
 				)
-				Expect(err).To(Not(HaveOccurred()))
 			}
 		})
 
@@ -85,14 +94,46 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator
 			}, tools.SetTimeout(3*time.Minute), 5*time.Second).Should(ContainSubstring("NotFound"))
 		})
 
-		By("Deleting cluster resource", func() {
-			Eventually(func() error {
-				_, err := kubectl.Run("delete", "cluster",
-					"--namespace", clusterNS, clusterName)
-				return err
-			}, tools.SetTimeout(2*time.Minute), 10*time.Second).Should(Not(HaveOccurred()))
+		// NOTE: we have to run this in background to be able to apply the workaround!
+		var wg sync.WaitGroup
+		wg.Add(1)
+		go func(ns, name string) {
+			defer wg.Done()
+			defer GinkgoRecover()
+
+			By("Deleting cluster resource", func() {
+				Eventually(func() error {
+					_, err := kubectl.Run("delete", "cluster",
+						"--namespace", ns, name)
+					return err
+				}, tools.SetTimeout(2*time.Minute), 10*time.Second).Should(Not(HaveOccurred()))
+			})
+		}(clusterNS, clusterName)
+
+		// Removing finalizers from MachineInventory and Machine
+		By("WORKAROUND: Removing finalizers from MachineInventory/Machine", func() {
+			// NOTE: wait a bit for the cluster deletion to be started (it's running in background)
+			time.Sleep(1 * time.Minute)
+
+			machineList, err := kubectl.Run("get", "MachineInventory",
+				"--namespace", clusterNS, "-o", "jsonpath={.items[*].metadata.name}")
+			Expect(err).To(Not(HaveOccurred()))
+
+			for _, machine := range strings.Fields(machineList) {
+				internalMachine, err := elemental.GetInternalMachine(clusterNS, machine)
+				Expect(err).To(Not(HaveOccurred()))
+
+				// Delete blocking Finalizers
+				GinkgoWriter.Printf("WORKAROUND EXECUTED: deleting Finalizers for MachineInventory '%s'...\n", machine)
+				deleteFinalizers(clusterNS, "MachineInventory", machine)
+				GinkgoWriter.Printf("WORKAROUND EXECUTED: deleting Finalizers for Machine '%s'...\n", internalMachine)
+				deleteFinalizers(clusterNS, "Machine", internalMachine)
+			}
 		})
 
+		// Wait for cluster deletion to be completed
+		wg.Wait()
+
 		By("Testing cluster resource unavailability", func() {
 			out, err := kubectl.Run("get", "cluster",
 				"--namespace", clusterNS, clusterName,
@@ -113,12 +154,12 @@ var _ = Describe("E2E - Uninstall Elemental Operator", Label("uninstall-operator
 						continue
 					}
 				}
-				err := kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart,
+				RunHelmCmdWithRetry("upgrade", "--install", chart,
 					operatorRepo+"/"+chart+"-chart",
 					"--namespace", "cattle-elemental-system",
 					"--create-namespace",
+					"--wait", "--wait-for-jobs",
 				)
-				Expect(err).To(Not(HaveOccurred()))
 			}
 
 			// Wait for pod to be started

tests/e2e/upgrade_test.go

@@ -54,12 +54,13 @@ var _ = Describe("E2E - Upgrading Elemental Operator", Label("upgrade-operator")
 		}
 
 		for _, chart := range upgradeOrder {
-			err := kubectl.RunHelmBinaryWithCustomErr("upgrade", "--install", chart,
+			RunHelmCmdWithRetry(
+				"upgrade", "--install", chart,
 				operatorUpgrade+"/"+chart+"-chart",
 				"--namespace", "cattle-elemental-system",
 				"--create-namespace",
+				"--wait", "--wait-for-jobs",
 			)
-			Expect(err).To(Not(HaveOccurred()))
 		}
 
 		// Delay few seconds before checking, needed because we may have 2 pods at the same time