🌱 Implement BMH provisioning E2E test

.gitignore

@@ -3,6 +3,9 @@ bin
 build/_output
 build/_test
 _artifacts
+artifacts.tar.gz
+test/e2e/images
+
 # Created by https://www.gitignore.io/api/go
 ### Go ###
 # Binaries for programs and plugins

Makefile

@@ -22,7 +22,7 @@ CRD_OPTIONS ?= "crd:allowDangerousTypes=true,crdVersions=v1"
 KUSTOMIZE = tools/bin/kustomize
 CONTROLLER_GEN = tools/bin/controller-gen
 GINKGO = tools/bin/ginkgo
-GINKGO_VER = v2.11.0
+GINKGO_VER = v2.12.0
 
 # See pkg/version.go for details
 SOURCE_GIT_COMMIT ?= $(shell git rev-parse --short HEAD)

hack/ci-e2e.sh

@@ -54,11 +54,36 @@ docker exec vbmc vbmc list
 # These variables are used by the tests. They override variables in the config file.
 # This IP is defined by the network we created above.
 # Together with the VBMC_PORT this becomes the BMC_ADDRESS used by the BMH in the test.
-export BMC_ADDRESS="ipmi://192.168.222.1:${VBMC_PORT}"
+IP_ADDRESS="192.168.222.1"
+export BMC_ADDRESS="ipmi://${IP_ADDRESS}:${VBMC_PORT}"
 export BOOT_MAC_ADDRESS
 # These are the VBMC defaults (used since we did not specify anything else for `vbmc add`).
 export BMC_USER=admin
 export BMC_PASSWORD=password
+CIRROS_VERSION="0.6.2"
+IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img"
+export IMAGE_CHECKSUM="c8fc807773e5354afe61636071771906"
+export IMAGE_URL="http://${IP_ADDRESS}/${IMAGE_FILE}"
+IMAGE_FOLDER="${REPO_ROOT}/test/e2e/images"
+
+## Setup image server
+# Create a directory for images
+
+mkdir -p "${IMAGE_FOLDER}"
+pushd "${IMAGE_FOLDER}"
+
+## Setup image server
+# Check if IMAGE_FILE already exists
+if [[ ! -f "${IMAGE_FILE}" ]]; then
+    wget "https://download.cirros-cloud.net/${CIRROS_VERSION}/${IMAGE_FILE}"
+else
+    echo "${IMAGE_FILE} already exists. Skipping download."
+fi
+
+popd
+
+# Run image server
+docker run --rm --name image-server-e2e -d -p 80:8080 -v "${IMAGE_FOLDER}:/usr/share/nginx/html" nginxinc/nginx-unprivileged
 
 # We need to gather artifacts/logs before exiting also if there are errors
 set +e

hack/clean-e2e.sh

@@ -4,11 +4,13 @@ REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 cd "${REPO_ROOT}" || exit 1
 
 minikube delete
-docker stop vbmc
-docker rm vbmc
+docker rm -f vbmc
+docker rm -f image-server-e2e
 virsh -c qemu:///system destroy --domain bmo-e2e-0
 virsh -c qemu:///system undefine --domain bmo-e2e-0 --remove-all-storage
 virsh -c qemu:///system net-destroy baremetal-e2e
 virsh -c qemu:///system net-undefine baremetal-e2e
 
-rm -rf "${REPO_ROOT}/test/e2e/_artifacts"
+rm -rfv "${REPO_ROOT}/test/e2e/_artifacts"
+rm -rfv "${REPO_ROOT}/artifacts.tar.gz"
+rm -rfv "${REPO_ROOT}/test/e2e/images"

test/e2e/basic_provisioning_test.go

@@ -0,0 +1,145 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/util"
+	"sigs.k8s.io/cluster-api/util/patch"
+
+	metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+)
+
+var _ = Describe("Provisioning", func() {
+	var (
+		specName       = "provisioning-ops"
+		namespace      *corev1.Namespace
+		cancelWatches  context.CancelFunc
+		bmcUser        string
+		bmcPassword    string
+		bmcAddress     string
+		bootMacAddress string
+	)
+
+	BeforeEach(func() {
+		bmcUser = e2eConfig.GetVariable("BMC_USER")
+		bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
+		bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
+		bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
+
+		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
+			Creator:   clusterProxy.GetClient(),
+			ClientSet: clusterProxy.GetClientSet(),
+			Name:      fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+			LogFolder: artifactFolder,
+		})
+	})
+
+	It("should provision and then deprovision a BMH", func() {
+		By("Creating a secret with BMH credentials")
+		bmcCredentials := corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      "bmc-credentials",
+				Namespace: namespace.Name,
+			},
+			StringData: map[string]string{
+				"username": bmcUser,
+				"password": bmcPassword,
+			},
+		}
+		err := clusterProxy.GetClient().Create(ctx, &bmcCredentials)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Creating a BMH with inspection disabled and hardware details added")
+		bmh := metal3api.BareMetalHost{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      specName,
+				Namespace: namespace.Name,
+				Annotations: map[string]string{
+					metal3api.InspectAnnotationPrefix:   "disabled",
+					metal3api.HardwareDetailsAnnotation: hardwareDetails,
+				},
+			},
+			Spec: metal3api.BareMetalHostSpec{
+				Online: true,
+				BMC: metal3api.BMCDetails{
+					Address:         bmcAddress,
+					CredentialsName: "bmc-credentials",
+				},
+				BootMode:       metal3api.Legacy,
+				BootMACAddress: bootMacAddress,
+			},
+		}
+		err = clusterProxy.GetClient().Create(ctx, &bmh)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Waiting for the BMH to be in registering state")
+		WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
+			Client: clusterProxy.GetClient(),
+			Bmh:    bmh,
+			State:  metal3api.StateRegistering,
+		}, e2eConfig.GetIntervals(specName, "wait-registering")...)
+
+		By("Waiting for the BMH to become available")
+		WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
+			Client: clusterProxy.GetClient(),
+			Bmh:    bmh,
+			State:  metal3api.StateAvailable,
+		}, e2eConfig.GetIntervals(specName, "wait-available")...)
+
+		By("Patching the BMH to test provisioning")
+		helper, err := patch.NewHelper(&bmh, clusterProxy.GetClient())
+		Expect(err).NotTo(HaveOccurred())
+		bmh.Spec.Image = &metal3api.Image{
+			URL:      e2eConfig.GetVariable("IMAGE_URL"),
+			Checksum: e2eConfig.GetVariable("IMAGE_CHECKSUM"),
+		}
+		bmh.Spec.RootDeviceHints = &metal3api.RootDeviceHints{
+			DeviceName: "/dev/vda",
+		}
+		Expect(helper.Patch(ctx, &bmh)).To(Succeed())
+
+		By("Waiting for the BMH to be in provisioning state")
+		WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
+			Client: clusterProxy.GetClient(),
+			Bmh:    bmh,
+			State:  metal3api.StateProvisioning,
+		}, e2eConfig.GetIntervals(specName, "wait-provisioning")...)
+
+		By("Waiting for the BMH to become provisioned")
+		WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
+			Client: clusterProxy.GetClient(),
+			Bmh:    bmh,
+			State:  metal3api.StateProvisioned,
+		}, e2eConfig.GetIntervals(specName, "wait-provisioned")...)
+
+		By("Triggering the deprovisioning of the BMH")
+		helper, err = patch.NewHelper(&bmh, clusterProxy.GetClient())
+		Expect(err).NotTo(HaveOccurred())
+		bmh.Spec.Image = nil
+		Expect(helper.Patch(ctx, &bmh)).To(Succeed())
+
+		By("Waiting for the BMH to be in deprovisioning state")
+		WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
+			Client: clusterProxy.GetClient(),
+			Bmh:    bmh,
+			State:  metal3api.StateDeprovisioning,
+		}, e2eConfig.GetIntervals(specName, "wait-deprovisioning")...)
+
+		By("Waiting for the BMH to become available again")
+		WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{
+			Client: clusterProxy.GetClient(),
+			Bmh:    bmh,
+			State:  metal3api.StateAvailable,
+		}, e2eConfig.GetIntervals(specName, "wait-available")...)
+	})
+
+	AfterEach(func() {
+		cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...)
+	})
+})

test/e2e/config/fixture.yaml

@@ -21,6 +21,8 @@ variables:
   BMC_PASSWORD: password
   BMC_ADDRESS: ipmi://192.168.222.1:16230
   BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
+  IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
+  IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
 
 intervals:
   inspection/wait-unmanaged: ["1m", "10ms"]
@@ -32,3 +34,8 @@ intervals:
   default/wait-deployment: ["5m", "1s"]
   default/wait-namespace-deleted: ["20s", "1s"]
   ironic/wait-deployment: ["10m", "2s"]
+  default/wait-registration: ["5s", "5ms"]
+  default/wait-provisioning: ["5s", "10ms"]
+  default/wait-provisioned: ["20s", "1s"]
+  default/wait-deprovisioning: ["5s", "10ms"]
+  default/wait-available: ["20s", "1s"]

test/e2e/config/ironic.yaml

@@ -24,6 +24,8 @@ variables:
   BMC_PASSWORD: password
   BMC_ADDRESS: ipmi://192.168.222.1:16230
   BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
+  IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
+  IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
 
 intervals:
   inspection/wait-unmanaged: ["1m", "5s"]
@@ -36,3 +38,6 @@ intervals:
   default/wait-namespace-deleted: ["10m", "1s"]
   ironic/wait-deployment: ["10m", "2s"]
   default/wait-power-state: ["10m", "100ms"]
+  default/wait-provisioning: ["20s", "1s"]
+  default/wait-provisioned: ["5m", "1s"]
+  default/wait-deprovisioning: ["1m", "10ms"]