From cbc8963dd7e1fc0fad6e084f5e0fa8e1e472cf99 Mon Sep 17 00:00:00 2001 From: Max Rantil Date: Tue, 24 Oct 2023 18:27:44 +0000 Subject: [PATCH] Implement BMH provisioning E2E test Introduce an end-to-end test validating the provisioning and deprovisioning flow of a BareMetalHost. --- Makefile | 2 +- hack/ci-e2e.sh | 25 ++++- hack/clean-e2e.sh | 7 +- test/e2e/basic_provisioning_test.go | 145 ++++++++++++++++++++++++++++ test/e2e/config/fixture.yaml | 5 + test/e2e/config/ironic.yaml | 5 + 6 files changed, 184 insertions(+), 5 deletions(-) create mode 100644 test/e2e/basic_provisioning_test.go diff --git a/Makefile b/Makefile index 3d965b4625..9f836789b0 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ CRD_OPTIONS ?= "crd:allowDangerousTypes=true,crdVersions=v1" KUSTOMIZE = tools/bin/kustomize CONTROLLER_GEN = tools/bin/controller-gen GINKGO = tools/bin/ginkgo -GINKGO_VER = v2.11.0 +GINKGO_VER = v2.12.0 # See pkg/version.go for details SOURCE_GIT_COMMIT ?= $(shell git rev-parse --short HEAD) diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index bad7997912..b44ead8fa4 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -54,11 +54,34 @@ docker exec vbmc vbmc list # These variables are used by the tests. They override variables in the config file. # This IP is defined by the network we created above. # Together with the VBMC_PORT this becomes the BMC_ADDRESS used by the BMH in the test. -export BMC_ADDRESS="ipmi://192.168.222.1:${VBMC_PORT}" +IP_ADDRESS="192.168.222.1" +export BMC_ADDRESS="ipmi://${IP_ADDRESS}:${VBMC_PORT}" export BOOT_MAC_ADDRESS # These are the VBMC defaults (used since we did not specify anything else for `vbmc add`). export BMC_USER=admin export BMC_PASSWORD=password +CIRROS_VERSION="0.6.2" +IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img" +export IMAGE_CHECKSUM="c8fc807773e5354afe61636071771906" +export IMAGE_URL="http://${IP_ADDRESS}/${IMAGE_FILE}" + +## Setup image server +# Create a directory for images +mkdir -p images +pushd images + +## Setup image server +# Check if IMAGE_FILE already exists +if [[ ! -f "${IMAGE_FILE}" ]]; then + wget "https://download.cirros-cloud.net/${CIRROS_VERSION}/${IMAGE_FILE}" +else + echo "${IMAGE_FILE} already exists. Skipping download." +fi + +popd + +# Run image server +docker run --rm --name image-server-e2e -d -p 80:8080 -v "$(pwd)/images:/usr/share/nginx/html" nginxinc/nginx-unprivileged # We need to gather artifacts/logs before exiting also if there are errors set +e diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh index f292b63ed4..ef8d533572 100755 --- a/hack/clean-e2e.sh +++ b/hack/clean-e2e.sh @@ -4,11 +4,12 @@ REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/.. cd "${REPO_ROOT}" || exit 1 minikube delete -docker stop vbmc -docker rm vbmc +docker rm -f vbmc +docker rm -f image-server-e2e virsh -c qemu:///system destroy --domain bmo-e2e-0 virsh -c qemu:///system undefine --domain bmo-e2e-0 --remove-all-storage virsh -c qemu:///system net-destroy baremetal-e2e virsh -c qemu:///system net-undefine baremetal-e2e -rm -rf "${REPO_ROOT}/test/e2e/_artifacts" +rm -rfv "${REPO_ROOT}/test/e2e/_artifacts" +rm -rfv "${REPO_ROOT}/images" diff --git a/test/e2e/basic_provisioning_test.go b/test/e2e/basic_provisioning_test.go new file mode 100644 index 0000000000..6d52e0160b --- /dev/null +++ b/test/e2e/basic_provisioning_test.go @@ -0,0 +1,145 @@ +package e2e + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/util" + "sigs.k8s.io/cluster-api/util/patch" + + metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" +) + +var _ = Describe("Provisioning", func() { + var ( + specName = "provisioning-ops" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmcUser string + bmcPassword string + bmcAddress string + bootMacAddress string + ) + + BeforeEach(func() { + bmcUser = e2eConfig.GetVariable("BMC_USER") + bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") + bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") + bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + + namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ + Creator: clusterProxy.GetClient(), + ClientSet: clusterProxy.GetClientSet(), + Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), + LogFolder: artifactFolder, + }) + }) + + It("should provision and then deprovision a BMH", func() { + By("Creating a secret with BMH credentials") + bmcCredentials := corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "bmc-credentials", + Namespace: namespace.Name, + }, + StringData: map[string]string{ + "username": bmcUser, + "password": bmcPassword, + }, + } + err := clusterProxy.GetClient().Create(ctx, &bmcCredentials) + Expect(err).NotTo(HaveOccurred()) + + By("Creating a BMH with inspection disabled and hardware details added") + bmh := metal3api.BareMetalHost{ + ObjectMeta: metav1.ObjectMeta{ + Name: specName, + Namespace: namespace.Name, + Annotations: map[string]string{ + metal3api.InspectAnnotationPrefix: "disabled", + metal3api.HardwareDetailsAnnotation: hardwareDetails, + }, + }, + Spec: metal3api.BareMetalHostSpec{ + Online: true, + BMC: metal3api.BMCDetails{ + Address: bmcAddress, + CredentialsName: "bmc-credentials", + }, + BootMode: metal3api.Legacy, + BootMACAddress: bootMacAddress, + }, + } + err = clusterProxy.GetClient().Create(ctx, &bmh) + Expect(err).NotTo(HaveOccurred()) + + By("Waiting for the BMH to be in registering state") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateRegistering, + }, e2eConfig.GetIntervals(specName, "wait-registering")...) + + By("Waiting for the BMH to become available") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateAvailable, + }, e2eConfig.GetIntervals(specName, "wait-available")...) + + By("Patching the BMH to test provisioning") + helper, err := patch.NewHelper(&bmh, clusterProxy.GetClient()) + Expect(err).NotTo(HaveOccurred()) + bmh.Spec.Image = &metal3api.Image{ + URL: e2eConfig.GetVariable("IMAGE_URL"), + Checksum: e2eConfig.GetVariable("IMAGE_CHECKSUM"), + } + bmh.Spec.RootDeviceHints = &metal3api.RootDeviceHints{ + DeviceName: "/dev/vda", + } + Expect(helper.Patch(ctx, &bmh)).To(Succeed()) + + By("Waiting for the BMH to be in provisioning state") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateProvisioning, + }, e2eConfig.GetIntervals(specName, "wait-provisioning")...) + + By("Waiting for the BMH to become provisioned") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateProvisioned, + }, e2eConfig.GetIntervals(specName, "wait-provisioned")...) + + By("Triggering the deprovisioning of the BMH") + helper, err = patch.NewHelper(&bmh, clusterProxy.GetClient()) + Expect(err).NotTo(HaveOccurred()) + bmh.Spec.Image = nil + Expect(helper.Patch(ctx, &bmh)).To(Succeed()) + + By("Waiting for the BMH to become deprovisioned") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateDeprovisioning, + }, e2eConfig.GetIntervals(specName, "wait-deprovisioning")...) + + By("Waiting for the BMH to become available again") + WaitForBmhInProvisioningState(ctx, WaitForBmhInProvisioningStateInput{ + Client: clusterProxy.GetClient(), + Bmh: bmh, + State: metal3api.StateAvailable, + }, e2eConfig.GetIntervals(specName, "wait-available")...) + }) + + AfterEach(func() { + cleanup(ctx, clusterProxy, namespace, cancelWatches, e2eConfig.GetIntervals("default", "wait-namespace-deleted")...) + }) +}) diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index 8688cc7992..adc871a3bd 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -21,6 +21,8 @@ variables: BMC_PASSWORD: password BMC_ADDRESS: ipmi://192.168.222.1:16230 BOOT_MAC_ADDRESS: "00:60:2f:31:81:01" + IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" + IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" intervals: inspection/wait-unmanaged: ["1m", "10ms"] @@ -32,3 +34,6 @@ intervals: default/wait-deployment: ["5m", "1s"] default/wait-namespace-deleted: ["20s", "1s"] ironic/wait-deployment: ["10m", "2s"] + default/wait-provisioning: ["20s", "1s"] + default/wait-provisioned: ["1m", "10ms"] + default/wait-deprovisioning: ["1m", "2s"] diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index c1bb07a20e..65074d8ced 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -24,6 +24,8 @@ variables: BMC_PASSWORD: password BMC_ADDRESS: ipmi://192.168.222.1:16230 BOOT_MAC_ADDRESS: "00:60:2f:31:81:01" + IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" + IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" intervals: inspection/wait-unmanaged: ["1m", "5s"] @@ -36,3 +38,6 @@ intervals: default/wait-namespace-deleted: ["10m", "1s"] ironic/wait-deployment: ["10m", "2s"] default/wait-power-state: ["10m", "100ms"] + default/wait-provisioning: ["20s", "1s"] + default/wait-provisioned: ["5m", "5s"] + default/wait-deprovisioning: ["1m", "10ms"]