Enable parallel testing in E2E

Makefile

@@ -48,7 +48,7 @@ export GOFLAGS=
 #
 GINKGO_FOCUS ?=
 GINKGO_SKIP ?=
-GINKGO_NODES ?= 1
+GINKGO_NODES ?= 2
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
 GINKGO_POLL_PROGRESS_INTERVAL ?= 5m

hack/ci-e2e.sh

@@ -58,38 +58,34 @@ minikube start
 minikube image load quay.io/metal3-io/baremetal-operator:e2e
 
 # Create libvirt domain
-VM_NAME="bmo-e2e-0"
-export BOOT_MAC_ADDRESS="00:60:2f:31:81:01"
+for vm_idx in $(seq 0 1); do
+  vm_name="bmo-e2e-${vm_idx}"
+  boot_mac_address="00:60:2f:31:81:0$(( vm_idx+1 ))"
 
-"${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${VM_NAME}" "${BOOT_MAC_ADDRESS}"
+  "${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${vm_name}" "${boot_mac_address}"
+done
 
 # This IP is defined by the network we created above.
 IP_ADDRESS="192.168.222.1"
 
-# These variables are used by the tests. They override variables in the config file.
-# These are the VBMC defaults (used since we did not specify anything else for `vbmc add`).
-export BMC_USER=admin
-export BMC_PASSWORD=password
-
 if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
-	# VBMC variables
-  VBMC_PORT="16230"
-  export BMC_ADDRESS="ipmi://${IP_ADDRESS}:${VBMC_PORT}"
-
   # Start VBMC
   docker run --name vbmc --network host -d \
     -v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
     -v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \
     quay.io/metal3-io/vbmc
 
-  # Add BMH VM to VBMC
-  "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${VM_NAME}" "${VBMC_PORT}"
+  for vm_idx in $(seq 0 1); do
+    vm_name="bmo-e2e-${vm_idx}"
+    vbmc_port="1623${vm_idx}"
+
+    # Add BMH VM to VBMC
+    "${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${vm_name}" "${vbmc_port}"
+  done
 
 elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
   # Sushy-tools variables
   SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf
-  SUSHY_PORT="8000"
-  export BMC_ADDRESS="redfish+http://${IP_ADDRESS}:${SUSHY_PORT}/redfish/v1/Systems/${VM_NAME}"
 
   # Start sushy-tools
   docker run --name sushy-tools -d --network host \
@@ -151,6 +147,8 @@ echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWOR
 echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
   "${IRONIC_INSPECTOR_PASSWORD}")" > "${IRONIC_OVERLAY}/ironic-inspector-htpasswd"
 
+export E2E_CONF_FILE=${REPO_ROOT}/test/e2e/config/ironic.yaml
+
 # We need to gather artifacts/logs before exiting also if there are errors
 set +e
 

hack/clean-e2e.sh

@@ -7,8 +7,11 @@ minikube delete
 docker rm -f vbmc
 docker rm -f image-server-e2e
 docker rm -f sushy-tools
+
 virsh -c qemu:///system destroy --domain bmo-e2e-0
 virsh -c qemu:///system undefine --domain bmo-e2e-0 --nvram --remove-all-storage
+virsh -c qemu:///system destroy --domain bmo-e2e-1
+virsh -c qemu:///system undefine --domain bmo-e2e-1 --nvram --remove-all-storage
 virsh -c qemu:///system net-destroy baremetal-e2e
 virsh -c qemu:///system net-undefine baremetal-e2e
 

hack/e2e/net.xml

@@ -11,6 +11,7 @@
       <range start='192.168.222.2' end='192.168.222.199'/>
       <host mac='52:54:00:6c:3c:01' name='minikube' ip='192.168.222.199'/>
       <host mac='00:60:2f:31:81:01' name='bmo-e2e-0' ip='192.168.222.122'/>
+      <host mac='00:60:2f:31:81:02' name='bmo-e2e-1' ip='192.168.222.123'/>
       <bootp file='http://192.168.222.199:6180/boot.ipxe'/>
     </dhcp>
   </ip>

test/e2e/basic_ops_test.go

@@ -23,20 +23,14 @@ var _ = Describe("basic", func() {
 		secretName     = "bmc-credentials"
 		namespace      *corev1.Namespace
 		cancelWatches  context.CancelFunc
-		bmcUser        string
-		bmcPassword    string
-		bmcAddress     string
-		bootMacAddress string
+		bmc            BMC
 	)
 	const (
 		rebootAnnotation   = "reboot.metal3.io"
 		poweroffAnnotation = "reboot.metal3.io/poweroff"
 	)
 	BeforeEach(func() {
-		bmcUser = e2eConfig.GetVariable("BMC_USER")
-		bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
-		bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
-		bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
+		bmc = e2eConfig.BMCs[GinkgoParallelProcess()-1]
 
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
 			Creator:   clusterProxy.GetClient(),
@@ -49,8 +43,8 @@ var _ = Describe("basic", func() {
 	It("should control power cycle of BMH though annotations", func() {
 		By("creating a secret with BMH credentials")
 		bmcCredentialsData := map[string]string{
-			"username": bmcUser,
-			"password": bmcPassword,
+			"username": bmc.User,
+			"password": bmc.Password,
 		}
 		CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)
 
@@ -66,11 +60,11 @@ var _ = Describe("basic", func() {
 			Spec: metal3api.BareMetalHostSpec{
 				Online: true,
 				BMC: metal3api.BMCDetails{
-					Address:         bmcAddress,
+					Address:         bmc.Address,
 					CredentialsName: "bmc-credentials",
 				},
 				BootMode:       metal3api.Legacy,
-				BootMACAddress: bootMacAddress,
+				BootMACAddress: bmc.BootMacAddress,
 			},
 		}
 		err := clusterProxy.GetClient().Create(ctx, &bmh)

test/e2e/common.go

@@ -53,12 +53,24 @@ const (
 	PoweredOff PowerState = "off"
 )
 
+type BMC struct {
+	User           string `yaml:"user,omitempty"`
+	Password       string `yaml:"password,omitempty"`
+	Address        string `yaml:"address,omitempty"`
+	BootMacAddress string `yaml:"bootMacAddress,omitempty"`
+	HostName       string `yaml:"hostName,omitempty"`
+	SSHAddress     string `yaml:"sshAddress,omitempty"`
+}
+
 // Config defines the configuration of an e2e test environment.
 type Config struct {
 	// Images is a list of container images to load into the Kind cluster.
 	// Note that this not relevant when using an existing cluster.
 	Images []clusterctl.ContainerImage `json:"images,omitempty"`
 
+	// BMHs objects to be used in the tests.
+	BMCs []BMC `json:"bmcs,omitempty"`
+
 	// Variables to be used in the tests.
 	Variables map[string]string `json:"variables,omitempty"`
 
@@ -391,13 +403,10 @@ echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData)
 // PerformSSHBootCheck performs an SSH check to verify the node's boot source.
 // The `expectedBootMode` parameter should be "disk" or "memory".
 // The `auth` parameter is an ssh.AuthMethod for authentication.
-func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod) {
-	ip := e2eConfig.GetVariable("IP_BMO_E2E_0")
-	sshPort := e2eConfig.GetVariable("SSH_PORT")
-	address := fmt.Sprintf("%s:%s", ip, sshPort)
+func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod, sshAddress string) {
 	user := e2eConfig.GetVariable("CIRROS_USERNAME")
 
-	client := EstablishSSHConnection(e2eConfig, auth, user, address)
+	client := EstablishSSHConnection(e2eConfig, auth, user, sshAddress)
 	defer func() {
 		if client != nil {
 			client.Close()

test/e2e/config/fixture.yaml

@@ -12,6 +12,22 @@ images:
 - name: quay.io/jetstack/cert-manager-controller:v1.13.1
   loadBehavior: tryLoad
 
+# Test credentials. The tests will create a BMH with these.
+# For the fixture provider it does not matter much what we put here.
+bmcs:
+- user: admin
+  password: password
+  address: ipmi://192.168.222.1:16230
+  boot_mac_address: "00:60:2f:31:81:01"
+  host_name: "bmo-e2e-0"
+  sshAddress: "192.168.222.122:22"
+- user: admin
+  password: password
+  address: ipmi://192.168.222.1:16231
+  boot_mac_address: "00:60:2f:31:81:02"
+  host_name: "bmo-e2e-1"
+  sshAddress: "192.168.222.123:22"
+
 variables:
   # These settings are for the "default" cluster, which is used for all tests except upgrade ones
   DEPLOY_IRONIC: "false"
@@ -25,13 +41,6 @@ variables:
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
   UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.4"
 
-  BMC_ADDRESS: ipmi://192.168.222.1:16230
-  BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
-  EXPECTED_HOST_NAME: ""
-  # Test credentials. The tests will create a BMH with these.
-  # For the fixture provider it does not matter much what we put here.
-  BMC_USER: admin
-  BMC_PASSWORD: password
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"

test/e2e/config/ironic.yaml

@@ -14,6 +14,26 @@ images:
 - name: quay.io/jetstack/cert-manager-controller:v1.13.1
   loadBehavior: tryLoad
 
+# Test credentials. The tests will create a BMH with these.
+# During development, select the preferred BMC interface when running tests from the command line:
+bmcs:
+- user: admin
+  password: password
+  # address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
+  # Uncomment the desired `address` line:
+  address: "ipmi://192.168.222.1:16230"
+  bootMacAddress: "00:60:2f:31:81:01"
+  hostName: "bmo-e2e-0"
+  sshAddress: "192.168.222.122:22"
+- user: admin
+  password: password
+  # address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1"
+  # Uncomment the desired `address` line:
+  address: "ipmi://192.168.222.1:16231"
+  bootMacAddress: "00:60:2f:31:81:02"
+  hostName: "bmo-e2e-1"
+  sshAddress: "192.168.222.123:22"
+
 # These variables can be overridden with environment variables.
 variables:
   # These settings are for the "default" cluster, which is used for all tests except upgrade ones
@@ -22,29 +42,17 @@ variables:
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/e2e"
   IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"
+
   # These settings are for the separate cluster that are used for upgrade tests
   UPGRADE_DEPLOY_IRONIC: "true"
   UPGRADE_DEPLOY_BMO: "true"
   UPGRADE_DEPLOY_CERT_MANAGER: "true"
   UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.4"
 
-  # Test credentials. The tests will create a BMH with these.
-  BMC_USER: admin
-  BMC_PASSWORD: password
-  # During development, select the preferred BMC interface when running tests from the command line:
-  # Note: These settings are only applicable for command line executions, not when running via the ci-e2e.sh script.
-  # Uncomment the desired BMC_ADDRESS line:
-  # BMC_ADDRESS: "ipmi://192.168.222.1:16230"
-  BMC_ADDRESS: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
-  BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
-  EXPECTED_HOST_NAME: "bmo-e2e-0"
-
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"
   SSH_CHECK_PROVISIONED: "true"
-  IP_BMO_E2E_0: "192.168.222.122"
-  SSH_PORT: "22"
   CIRROS_USERNAME: "cirros"
   CIRROS_PASSWORD: "gocubsgo"
   SSH_PRIV_KEY: "./images/ssh_testkey"

test/e2e/external_inspection_test.go

@@ -177,16 +177,10 @@ var _ = Describe("External Inspection", func() {
 		secretName     = "bmc-credentials"
 		namespace      *corev1.Namespace
 		cancelWatches  context.CancelFunc
-		bmcUser        string
-		bmcPassword    string
-		bmcAddress     string
-		bootMacAddress string
+		bmc            BMC
 	)
 	BeforeEach(func() {
-		bmcUser = e2eConfig.GetVariable("BMC_USER")
-		bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
-		bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
-		bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
+		bmc = e2eConfig.BMCs[GinkgoParallelProcess()-1]
 
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
 			Creator:   clusterProxy.GetClient(),
@@ -199,8 +193,8 @@ var _ = Describe("External Inspection", func() {
 	It("should skip inspection and become available when a BMH has annotations with hardware details and inspection disabled", func() {
 		By("Creating a secret with BMH credentials")
 		bmcCredentialsData := map[string]string{
-			"username": bmcUser,
-			"password": bmcPassword,
+			"username": bmc.User,
+			"password": bmc.Password,
 		}
 		CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)
 
@@ -216,11 +210,11 @@ var _ = Describe("External Inspection", func() {
 			},
 			Spec: metal3api.BareMetalHostSpec{
 				BMC: metal3api.BMCDetails{
-					Address:         bmcAddress,
+					Address:         bmc.Address,
 					CredentialsName: "bmc-credentials",
 				},
 				BootMode:       metal3api.Legacy,
-				BootMACAddress: bootMacAddress,
+				BootMACAddress: bmc.BootMacAddress,
 			},
 		}
 		err := clusterProxy.GetClient().Create(ctx, &bmh)

test/e2e/inspection_test.go

@@ -21,16 +21,11 @@ var _ = Describe("Inspection", func() {
 		secretName     = "bmc-credentials"
 		namespace      *corev1.Namespace
 		cancelWatches  context.CancelFunc
-		bmcUser        string
-		bmcPassword    string
-		bmcAddress     string
-		bootMacAddress string
+		bmc            BMC
 	)
 	BeforeEach(func() {
-		bmcUser = e2eConfig.GetVariable("BMC_USER")
-		bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
-		bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
-		bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
+		bmc = e2eConfig.BMCs[GinkgoParallelProcess()-1]
+
 
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
 			Creator:   clusterProxy.GetClient(),
@@ -93,9 +88,10 @@ var _ = Describe("Inspection", func() {
 
 	It("should inspect a newly created BMH", func() {
 		By("Creating a secret with BMH credentials")
+
 		bmcCredentialsData := map[string]string{
-			"username": bmcUser,
-			"password": bmcPassword,
+			"username": bmc.User,
+			"password": bmc.Password,
 		}
 		CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)
 
@@ -107,11 +103,11 @@ var _ = Describe("Inspection", func() {
 			},
 			Spec: metal3api.BareMetalHostSpec{
 				BMC: metal3api.BMCDetails{
-					Address:         bmcAddress,
+					Address:         bmc.Address,
 					CredentialsName: "bmc-credentials",
 				},
 				BootMode:       metal3api.Legacy,
-				BootMACAddress: bootMacAddress,
+				BootMACAddress: bmc.BootMacAddress,
 			},
 		}
 		err := clusterProxy.GetClient().Create(ctx, &bmh)