From f57b2ba75fedae3d4806e3a1b008cc1831b61b41 Mon Sep 17 00:00:00 2001 From: Huy Mai Date: Mon, 22 Jan 2024 14:51:39 +0000 Subject: [PATCH] Enable parallel testing in E2E Signed-off-by: Huy Mai --- Makefile | 6 ++-- hack/ci-e2e.sh | 33 ++++++++--------- hack/clean-e2e.sh | 14 ++++++-- hack/create_bmcs.sh | 37 ++++++++++++++++++++ hack/e2e/net.xml | 1 - test/e2e/basic_ops_test.go | 26 ++++++-------- test/e2e/bmc.go | 27 ++++++++++++++ test/e2e/common.go | 7 ++-- test/e2e/config/bmcs-fixture.yaml | 30 ++++++++++++++++ test/e2e/config/bmcs-sushy-tools.yaml | 21 +++++++++++ test/e2e/config/bmcs-vbmc.yaml | 21 +++++++++++ test/e2e/config/fixture.yaml | 11 ++---- test/e2e/config/ironic.yaml | 14 +------- test/e2e/e2e_suite_test.go | 8 +++++ test/e2e/external_inspection_test.go | 26 ++++++-------- test/e2e/inspection_test.go | 27 ++++++-------- test/e2e/live_iso_test.go | 31 +++++++--------- test/e2e/provisioning_and_annotation_test.go | 36 ++++++++----------- test/e2e/re_inspection_test.go | 30 ++++++---------- test/e2e/upgrade_test.go | 19 ++++------ 20 files changed, 255 insertions(+), 170 deletions(-) create mode 100755 hack/create_bmcs.sh create mode 100644 test/e2e/bmc.go create mode 100644 test/e2e/config/bmcs-fixture.yaml create mode 100644 test/e2e/config/bmcs-sushy-tools.yaml create mode 100644 test/e2e/config/bmcs-vbmc.yaml diff --git a/Makefile b/Makefile index 8d82b954b7..b4cf5b3361 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,12 @@ export GOFLAGS= # GINKGO_FOCUS ?= GINKGO_SKIP ?= -GINKGO_NODES ?= 1 +GINKGO_NODES ?= 4 GINKGO_TIMEOUT ?= 2h GINKGO_POLL_PROGRESS_AFTER ?= 60m GINKGO_POLL_PROGRESS_INTERVAL ?= 5m E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/fixture.yaml +E2E_BMCS_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/bmcs-fixture.yaml USE_EXISTING_CLUSTER ?= false SKIP_RESOURCE_CLEANUP ?= false GINKGO_NOCOLOR ?= false @@ -118,7 +119,8 @@ test-e2e: $(GINKGO) ## Run the end-to-end tests -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) test/e2e -- \ - -e2e.config="$(E2E_CONF_FILE)" -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \ + -e2e.config="$(E2E_CONF_FILE)" -e2e.bmcsConfig="$(E2E_BMCS_CONF_FILE)" \ + -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \ -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.artifacts-folder="$(ARTIFACTS)" ## -------------------------------------- diff --git a/hack/ci-e2e.sh b/hack/ci-e2e.sh index d8990830f5..bb6b489193 100755 --- a/hack/ci-e2e.sh +++ b/hack/ci-e2e.sh @@ -59,39 +59,20 @@ minikube start # Load the BMO e2e image into it minikube image load quay.io/metal3-io/baremetal-operator:e2e -# Create libvirt domain -VM_NAME="bmo-e2e-0" -export BOOT_MAC_ADDRESS="00:60:2f:31:81:01" - -"${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${VM_NAME}" "${BOOT_MAC_ADDRESS}" - # This IP is defined by the network we created above. IP_ADDRESS="192.168.222.1" -# These variables are used by the tests. They override variables in the config file. -# These are the VBMC defaults (used since we did not specify anything else for `vbmc add`). -export BMC_USER=admin -export BMC_PASSWORD=password - if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then - # VBMC variables - VBMC_PORT="16230" - export BMC_ADDRESS="ipmi://${IP_ADDRESS}:${VBMC_PORT}" - # Start VBMC docker run --name vbmc --network host -d \ -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \ -v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \ quay.io/metal3-io/vbmc - # Add BMH VM to VBMC - "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${VM_NAME}" "${VBMC_PORT}" elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then # Sushy-tools variables SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf - SUSHY_PORT="8000" - export BMC_ADDRESS="redfish+http://${IP_ADDRESS}:${SUSHY_PORT}/redfish/v1/Systems/${VM_NAME}" # Start sushy-tools docker run --name sushy-tools -d --network host \ @@ -105,6 +86,19 @@ else exit 1 fi +export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml" +"${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}" + +# Set the number of ginkgo processes to the smaller number in +# the number of host CPUs and the number of BMCs +n_cpus=$(nproc) +n_vms=$(yq '. | length' "${E2E_BMCS_CONF_FILE}") +if [ "${n_cpus}" -lt "${n_vms}" ]; then + export GINKGO_NODES="${n_cpus}" +else + export GINKGO_NODES="${n_vms}" +fi + # Image server variables CIRROS_VERSION="0.6.2" IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img" @@ -153,6 +147,7 @@ echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWOR echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \ "${IRONIC_INSPECTOR_PASSWORD}")" > "${IRONIC_OVERLAY}/ironic-inspector-htpasswd" + # We need to gather artifacts/logs before exiting also if there are errors set +e diff --git a/hack/clean-e2e.sh b/hack/clean-e2e.sh index 06d29ceb09..9bb8acf3bf 100755 --- a/hack/clean-e2e.sh +++ b/hack/clean-e2e.sh @@ -7,11 +7,19 @@ minikube delete docker rm -f vbmc docker rm -f image-server-e2e docker rm -f sushy-tools -virsh -c qemu:///system destroy --domain bmo-e2e-0 -virsh -c qemu:///system undefine --domain bmo-e2e-0 --nvram --remove-all-storage + +virsh_vms=$(virsh list --name --all) + +for vm in ${virsh_vms}; do + if [[ "${vm}" =~ "bmo-e2e-" ]]; then + virsh -c qemu:///system destroy --domain "${vm}" + virsh -c qemu:///system undefine --domain "${vm}" --nvram --remove-all-storage + fi +done + virsh -c qemu:///system net-destroy baremetal-e2e virsh -c qemu:///system net-undefine baremetal-e2e rm -rf "${REPO_ROOT}/test/e2e/_artifacts" -rm -rf "${REPO_ROOT}/artifacts.tar.gz" +rm -rf "${REPO_ROOT}"/artifacts-* rm -rf "${REPO_ROOT}/test/e2e/images" diff --git a/hack/create_bmcs.sh b/hack/create_bmcs.sh new file mode 100755 index 0000000000..d40a9312f6 --- /dev/null +++ b/hack/create_bmcs.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# This script reads BMC information in a config file and prepare VMs +# whose info match those config +# +set -eux + +REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..") +cd "${REPO_ROOT}" || exit 1 + +CONFIG_FILE=$1 +NETWORK=${2:-"baremetal-e2e"} + +readarray -t BMCs < <(yq e -o=j -I=0 '.[]' "${CONFIG_FILE}") + +# N_BMCS=$(yq '.bmcs | length' "${CONFIG_FILE}") +# echo ${N_BMCS} +for bmc in "${BMCs[@]}"; do + address=$(echo "${bmc}" | jq -r '.address') + bootMacAddress=$(echo "${bmc}" | jq -r '.bootMacAddress') + hostName=$(echo "${bmc}" | jq -r '.hostName') + ipAddress=$(echo "${bmc}" | jq -r '.ipAddress') + + # Create VM + "${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${hostName}" "${bootMacAddress}" + + # Add the the VM to the network host list + virsh net-update "${NETWORK}" add-last ip-dhcp-host \ + "" \ + --live --config --parent-index 0 + + # Add BMH VM to VBMC + if [[ "${address}" =~ "ipmi://" ]]; then + vbmc_port="${address##*:}" + "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}" + fi +done diff --git a/hack/e2e/net.xml b/hack/e2e/net.xml index 54a2740bfb..71fdad47e8 100644 --- a/hack/e2e/net.xml +++ b/hack/e2e/net.xml @@ -10,7 +10,6 @@ - diff --git a/test/e2e/basic_ops_test.go b/test/e2e/basic_ops_test.go index e06a965670..8a7fd50007 100644 --- a/test/e2e/basic_ops_test.go +++ b/test/e2e/basic_ops_test.go @@ -19,24 +19,18 @@ import ( var _ = Describe("basic", func() { var ( - specName = "basic-ops" - secretName = "bmc-credentials" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string + specName = "basic-ops" + secretName = "bmc-credentials" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmc BMC ) const ( rebootAnnotation = "reboot.metal3.io" poweroffAnnotation = "reboot.metal3.io/poweroff" ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + bmc = (*bmcs)[GinkgoParallelProcess()-1] namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), @@ -49,8 +43,8 @@ var _ = Describe("basic", func() { It("should control power cycle of BMH though annotations", func() { By("creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -66,11 +60,11 @@ var _ = Describe("basic", func() { Spec: metal3api.BareMetalHostSpec{ Online: true, BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: "bmc-credentials", }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, }, } err := clusterProxy.GetClient().Create(ctx, &bmh) diff --git a/test/e2e/bmc.go b/test/e2e/bmc.go new file mode 100644 index 0000000000..d49d913335 --- /dev/null +++ b/test/e2e/bmc.go @@ -0,0 +1,27 @@ +package e2e + +import ( + "os" + + . "github.com/onsi/gomega" + "gopkg.in/yaml.v2" +) + +// BMC defines a BMH to use in the tests +type BMC struct { + User string `yaml:"user,omitempty"` + Password string `yaml:"password,omitempty"` + Address string `yaml:"address,omitempty"` + BootMacAddress string `yaml:"bootMacAddress,omitempty"` + HostName string `yaml:"hostName,omitempty"` + IpAddress string `yaml:"ipAddress,omitempty"` + SSHPort string `yaml:"sshPort,omitempty"` +} + +func LoadBMCConfig(configPath string) *[]BMC { + configData, err := os.ReadFile(configPath) //#nosec + Expect(err).ToNot(HaveOccurred(), "Failed to read the bmcs config file") + var bmcs []BMC + Expect(yaml.Unmarshal(configData, &bmcs)).To(Succeed()) + return &bmcs +} diff --git a/test/e2e/common.go b/test/e2e/common.go index 32ae8a248c..cc5c520a1e 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -391,13 +391,10 @@ echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData) // PerformSSHBootCheck performs an SSH check to verify the node's boot source. // The `expectedBootMode` parameter should be "disk" or "memory". // The `auth` parameter is an ssh.AuthMethod for authentication. -func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod) { - ip := e2eConfig.GetVariable("IP_BMO_E2E_0") - sshPort := e2eConfig.GetVariable("SSH_PORT") - address := fmt.Sprintf("%s:%s", ip, sshPort) +func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod, sshAddress string) { user := e2eConfig.GetVariable("CIRROS_USERNAME") - client := EstablishSSHConnection(e2eConfig, auth, user, address) + client := EstablishSSHConnection(e2eConfig, auth, user, sshAddress) defer func() { if client != nil { client.Close() diff --git a/test/e2e/config/bmcs-fixture.yaml b/test/e2e/config/bmcs-fixture.yaml new file mode 100644 index 0000000000..bdc0f3b610 --- /dev/null +++ b/test/e2e/config/bmcs-fixture.yaml @@ -0,0 +1,30 @@ +# For fixture config, only the `hostName` field matters. The number of BMCs, +# however, needs to be equal or larger than `GINKGO_NODES`. +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0" + bootMacAddress: "00:60:2f:31:81:01" + hostName: "" + ipAddress: "192.168.222.122" + sshPort: "22" +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1" + bootMacAddress: "00:60:2f:31:81:02" + hostName: "" + ipAddress: "192.168.222.123" + sshPort: "22" +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-2" + bootMacAddress: "00:60:2f:31:81:01" + hostName: "" + ipAddress: "192.168.222.124" + sshPort: "22" +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-3" + bootMacAddress: "00:60:2f:31:81:02" + hostName: "" + ipAddress: "192.168.222.125" + sshPort: "22" diff --git a/test/e2e/config/bmcs-sushy-tools.yaml b/test/e2e/config/bmcs-sushy-tools.yaml new file mode 100644 index 0000000000..dde99bef83 --- /dev/null +++ b/test/e2e/config/bmcs-sushy-tools.yaml @@ -0,0 +1,21 @@ +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0" + bootMacAddress: "00:60:2f:31:81:01" + hostName: "bmo-e2e-0" + ipAddress: "192.168.222.122" + sshPort: "22" +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1" + bootMacAddress: "00:60:2f:31:81:02" + hostName: "bmo-e2e-1" + ipAddress: "192.168.222.123" + sshPort: "22" +- user: admin + password: password + address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-2" + bootMacAddress: "00:60:2f:31:81:03" + hostName: "bmo-e2e-2" + ipAddress: "192.168.222.124" + sshPort: "22" diff --git a/test/e2e/config/bmcs-vbmc.yaml b/test/e2e/config/bmcs-vbmc.yaml new file mode 100644 index 0000000000..c8f32ad70b --- /dev/null +++ b/test/e2e/config/bmcs-vbmc.yaml @@ -0,0 +1,21 @@ +- user: admin + password: password + address: "ipmi://192.168.222.1:16230" + bootMacAddress: "00:60:2f:31:81:01" + hostName: "bmo-e2e-0" + ipAddress: "192.168.222.122" + sshPort: "22" +- user: admin + password: password + address: "ipmi://192.168.222.1:16231" + bootMacAddress: "00:60:2f:31:81:02" + hostName: "bmo-e2e-1" + ipAddress: "192.168.222.123" + sshPort: "22" +- user: admin + password: password + address: "ipmi://192.168.222.1:16232" + bootMacAddress: "00:60:2f:31:81:03" + hostName: "bmo-e2e-2" + ipAddress: "192.168.222.124" + sshPort: "22" diff --git a/test/e2e/config/fixture.yaml b/test/e2e/config/fixture.yaml index aeccb77c0f..c50da6a5fd 100644 --- a/test/e2e/config/fixture.yaml +++ b/test/e2e/config/fixture.yaml @@ -25,13 +25,6 @@ variables: UPGRADE_DEPLOY_CERT_MANAGER: "true" UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.4" - BMC_ADDRESS: ipmi://192.168.222.1:16230 - BOOT_MAC_ADDRESS: "00:60:2f:31:81:01" - EXPECTED_HOST_NAME: "" - # Test credentials. The tests will create a BMH with these. - # For the fixture provider it does not matter much what we put here. - BMC_USER: admin - BMC_PASSWORD: password IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" CERT_MANAGER_VERSION: "v1.13.1" @@ -45,11 +38,11 @@ intervals: inspection/wait-available: ["5s", "1ms"] external-inspection/wait-available: ["5s", "1ms"] default/wait-deployment: ["5m", "1s"] - default/wait-namespace-deleted: ["20s", "1s"] + default/wait-namespace-deleted: ["30s", "1s"] ironic/wait-deployment: ["10m", "2s"] default/wait-registration: ["5s", "5ms"] default/wait-provisioning: ["30s", "10ms"] - default/wait-provisioned: ["1m", "1s"] + default/wait-provisioned: ["2m", "1s"] default/wait-deprovisioning: ["5s", "10ms"] default/wait-available: ["2m", "1s"] default/wait-deleting: ["5s", "10ms"] diff --git a/test/e2e/config/ironic.yaml b/test/e2e/config/ironic.yaml index 26911548b1..30cc5201ca 100644 --- a/test/e2e/config/ironic.yaml +++ b/test/e2e/config/ironic.yaml @@ -22,29 +22,17 @@ variables: DEPLOY_CERT_MANAGER: "true" BMO_KUSTOMIZATION: "../../config/overlays/e2e" IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e" + # These settings are for the separate cluster that are used for upgrade tests UPGRADE_DEPLOY_IRONIC: "true" UPGRADE_DEPLOY_BMO: "true" UPGRADE_DEPLOY_CERT_MANAGER: "true" UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.4" - # Test credentials. The tests will create a BMH with these. - BMC_USER: admin - BMC_PASSWORD: password - # During development, select the preferred BMC interface when running tests from the command line: - # Note: These settings are only applicable for command line executions, not when running via the ci-e2e.sh script. - # Uncomment the desired BMC_ADDRESS line: - # BMC_ADDRESS: "ipmi://192.168.222.1:16230" - BMC_ADDRESS: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0" - BOOT_MAC_ADDRESS: "00:60:2f:31:81:01" - EXPECTED_HOST_NAME: "bmo-e2e-0" - IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img" IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906" CERT_MANAGER_VERSION: "v1.13.1" SSH_CHECK_PROVISIONED: "true" - IP_BMO_E2E_0: "192.168.222.122" - SSH_PORT: "22" CIRROS_USERNAME: "cirros" CIRROS_PASSWORD: "gocubsgo" SSH_PRIV_KEY: "./images/ssh_testkey" diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 4615944ded..e1eda13af3 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -27,12 +27,18 @@ var ( // configPath is the path to the e2e config file. configPath string + // bmcConfigPath is the path to the file whose content is the list of bmcs used in the test + bmcConfigPath string + // useExistingCluster instructs the test to use the current cluster instead of creating a new one (default discovery rules apply). useExistingCluster bool // e2eConfig to be used for this test, read from configPath. e2eConfig *Config + // bmcs to be used for this test, read from bmcConfigPath + bmcs *[]BMC + // artifactFolder is the folder to store e2e test artifacts. artifactFolder string @@ -49,6 +55,7 @@ var ( func init() { flag.StringVar(&configPath, "e2e.config", "", "path to the e2e config file") + flag.StringVar(&bmcConfigPath, "e2e.bmcsConfig", "", "path to the bmcs config file") flag.StringVar(&artifactFolder, "e2e.artifacts-folder", "_artifacts", "folder where e2e test artifact should be stored") flag.BoolVar(&useExistingCluster, "e2e.use-existing-cluster", false, "if true, the test uses the current cluster instead of creating a new one (default discovery rules apply)") flag.BoolVar(&skipCleanup, "e2e.skip-resource-cleanup", false, "if true, the resource cleanup after tests will be skipped") @@ -161,6 +168,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { metal3api.AddToScheme(scheme) e2eConfig = LoadE2EConfig(configPath) + bmcs = LoadBMCConfig(bmcConfigPath) clusterProxy = framework.NewClusterProxy("bmo-e2e", kubeconfigPath, scheme) }) diff --git a/test/e2e/external_inspection_test.go b/test/e2e/external_inspection_test.go index 03a494a5c9..76a6584d49 100644 --- a/test/e2e/external_inspection_test.go +++ b/test/e2e/external_inspection_test.go @@ -173,20 +173,14 @@ const hardwareDetails = ` var _ = Describe("External Inspection", func() { var ( - specName = "external-inspection" - secretName = "bmc-credentials" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string + specName = "external-inspection" + secretName = "bmc-credentials" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmc BMC ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + bmc = (*bmcs)[GinkgoParallelProcess()-1] namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), @@ -199,8 +193,8 @@ var _ = Describe("External Inspection", func() { It("should skip inspection and become available when a BMH has annotations with hardware details and inspection disabled", func() { By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -216,11 +210,11 @@ var _ = Describe("External Inspection", func() { }, Spec: metal3api.BareMetalHostSpec{ BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: "bmc-credentials", }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, }, } err := clusterProxy.GetClient().Create(ctx, &bmh) diff --git a/test/e2e/inspection_test.go b/test/e2e/inspection_test.go index 552aabd790..eaa65fb227 100644 --- a/test/e2e/inspection_test.go +++ b/test/e2e/inspection_test.go @@ -17,20 +17,14 @@ import ( var _ = Describe("Inspection", func() { var ( - specName = "inspection" - secretName = "bmc-credentials" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string + specName = "inspection" + secretName = "bmc-credentials" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmc BMC ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + bmc = (*bmcs)[GinkgoParallelProcess()-1] namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), @@ -93,9 +87,10 @@ var _ = Describe("Inspection", func() { It("should inspect a newly created BMH", func() { By("Creating a secret with BMH credentials") + bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -107,11 +102,11 @@ var _ = Describe("Inspection", func() { }, Spec: metal3api.BareMetalHostSpec{ BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: "bmc-credentials", }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, }, } err := clusterProxy.GetClient().Create(ctx, &bmh) diff --git a/test/e2e/live_iso_test.go b/test/e2e/live_iso_test.go index fc8bb3a2c3..69f09d445a 100644 --- a/test/e2e/live_iso_test.go +++ b/test/e2e/live_iso_test.go @@ -22,22 +22,17 @@ import ( var _ = Describe("Live-ISO", func() { var ( - specName = "live-iso-ops" - secretName = "bmc-credentials" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string - imageURL string + specName = "live-iso-ops" + secretName = "bmc-credentials" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmc BMC + imageURL string ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + bmc = (*bmcs)[GinkgoParallelProcess()-1] + imageURL = e2eConfig.GetVariable("IMAGE_URL") namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ @@ -51,8 +46,8 @@ var _ = Describe("Live-ISO", func() { It("should provision a BMH with live ISO and then deprovision it", func() { By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -69,7 +64,7 @@ var _ = Describe("Live-ISO", func() { Spec: metal3api.BareMetalHostSpec{ Online: true, BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: secretName, }, Image: &metal3api.Image{ @@ -77,7 +72,7 @@ var _ = Describe("Live-ISO", func() { DiskFormat: pointer.String("live-iso"), }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, }, } err := clusterProxy.GetClient().Create(ctx, &bmh) @@ -102,7 +97,7 @@ var _ = Describe("Live-ISO", func() { By("Verifying the node booted from live ISO image") password := e2eConfig.GetVariable("CIRROS_PASSWORD") auth := ssh.Password(password) - PerformSSHBootCheck(e2eConfig, "memory", auth) + PerformSSHBootCheck(e2eConfig, "memory", auth, fmt.Sprintf("%s:%s", bmc.IpAddress, bmc.SSHPort)) } else { capm3_e2e.Logf("WARNING: Skipping SSH check since SSH_CHECK_PROVISIONED != true") } diff --git a/test/e2e/provisioning_and_annotation_test.go b/test/e2e/provisioning_and_annotation_test.go index 2f8294926f..a5132bf6f7 100644 --- a/test/e2e/provisioning_and_annotation_test.go +++ b/test/e2e/provisioning_and_annotation_test.go @@ -26,21 +26,15 @@ import ( var _ = Describe("BMH Provisioning and Annotation Management", func() { var ( - specName = "provisioning-ops" - secretName = "bmc-credentials" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string + specName = "provisioning-ops" + secretName = "bmc-credentials" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmc BMC ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + bmc = (*bmcs)[GinkgoParallelProcess()-1] namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), @@ -53,8 +47,8 @@ var _ = Describe("BMH Provisioning and Annotation Management", func() { It("provisions a BMH, applies detached and status annotations, then deprovisions", func() { By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -71,11 +65,11 @@ var _ = Describe("BMH Provisioning and Annotation Management", func() { Spec: metal3api.BareMetalHostSpec{ Online: true, BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: "bmc-credentials", }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, AutomatedCleaningMode: "disabled", }, } @@ -137,7 +131,7 @@ var _ = Describe("BMH Provisioning and Annotation Management", func() { Expect(err).NotTo(HaveOccurred(), "unable to parse private key") auth := ssh.PublicKeys(signer) - PerformSSHBootCheck(e2eConfig, "disk", auth) + PerformSSHBootCheck(e2eConfig, "disk", auth, fmt.Sprintf("%s:%s", bmc.IpAddress, bmc.SSHPort)) } else { capm3_e2e.Logf("WARNING: Skipping SSH check since SSH_CHECK_PROVISIONED != true") } @@ -194,8 +188,8 @@ var _ = Describe("BMH Provisioning and Annotation Management", func() { By("Creating a secret with BMH credentials") bmcCredentialsData = map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -211,11 +205,11 @@ var _ = Describe("BMH Provisioning and Annotation Management", func() { Spec: metal3api.BareMetalHostSpec{ Online: true, BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: "bmc-credentials", }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, AutomatedCleaningMode: "disabled", Image: &metal3api.Image{ URL: e2eConfig.GetVariable("IMAGE_URL"), diff --git a/test/e2e/re_inspection_test.go b/test/e2e/re_inspection_test.go index 1b1f469cbb..50e9b2a269 100644 --- a/test/e2e/re_inspection_test.go +++ b/test/e2e/re_inspection_test.go @@ -21,25 +21,17 @@ import ( var _ = Describe("Re-Inspection", func() { var ( - specName = "re-inspection" - secretName = "bmc-credentials" - namespace *corev1.Namespace - cancelWatches context.CancelFunc - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string - expectedHostName string + specName = "re-inspection" + secretName = "bmc-credentials" + namespace *corev1.Namespace + cancelWatches context.CancelFunc + bmc BMC ) const ( wrongHostName = "wrongHostName" ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") - expectedHostName = e2eConfig.GetVariable("EXPECTED_HOST_NAME") + bmc = (*bmcs)[GinkgoParallelProcess()-1] namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ Creator: clusterProxy.GetClient(), @@ -52,8 +44,8 @@ var _ = Describe("Re-Inspection", func() { It("should re-inspect the annotated BMH", func() { By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -70,11 +62,11 @@ var _ = Describe("Re-Inspection", func() { }, Spec: metal3api.BareMetalHostSpec{ BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: "bmc-credentials", }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, }, } err := clusterProxy.GetClient().Create(ctx, &bmh) @@ -115,7 +107,7 @@ var _ = Describe("Re-Inspection", func() { By("checking that the hardware details are corrected after re-inspection") key = types.NamespacedName{Namespace: bmh.Namespace, Name: bmh.Name} Expect(clusterProxy.GetClient().Get(ctx, key, &bmh)).To(Succeed()) - Expect(bmh.Status.HardwareDetails.Hostname).To(Equal(expectedHostName)) + Expect(bmh.Status.HardwareDetails.Hostname).To(Equal(bmc.HostName)) }) AfterEach(func() { diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index 6a2d94878c..d841e19e2f 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -175,20 +175,15 @@ var _ = Describe("BMO Upgrade", func() { specName = "upgrade" secretName = "bmc-credentials" namespace *corev1.Namespace - bmcUser string - bmcPassword string - bmcAddress string - bootMacAddress string + bmc BMC bmoIronicNamespace string upgradeClusterProvider bootstrap.ClusterProvider upgradeClusterProxy framework.ClusterProxy bmh metal3api.BareMetalHost ) BeforeEach(func() { - bmcUser = e2eConfig.GetVariable("BMC_USER") - bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD") - bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS") - bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS") + bmc = (*bmcs)[GinkgoParallelProcess()-1] + bmoIronicNamespace = "baremetal-operator-system" var kubeconfigPath string @@ -266,8 +261,8 @@ var _ = Describe("BMO Upgrade", func() { It("Should upgrade BMO to latest version", func() { By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ - "username": bmcUser, - "password": bmcPassword, + "username": bmc.User, + "password": bmc.Password, } CreateSecret(ctx, upgradeClusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData) @@ -284,11 +279,11 @@ var _ = Describe("BMO Upgrade", func() { Spec: metal3api.BareMetalHostSpec{ Online: true, BMC: metal3api.BMCDetails{ - Address: bmcAddress, + Address: bmc.Address, CredentialsName: secretName, }, BootMode: metal3api.Legacy, - BootMACAddress: bootMacAddress, + BootMACAddress: bmc.BootMacAddress, }, } err := upgradeClusterProxy.GetClient().Create(ctx, &bmh)