Skip to content

Commit

Permalink
Enable parallel testing in E2E
Browse files Browse the repository at this point in the history
Signed-off-by: Huy Mai <huy.mai@est.tech>
  • Loading branch information
mquhuy committed Jan 24, 2024
1 parent 8b412ce commit f57b2ba
Show file tree
Hide file tree
Showing 20 changed files with 255 additions and 170 deletions.
6 changes: 4 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,12 @@ export GOFLAGS=
#
GINKGO_FOCUS ?=
GINKGO_SKIP ?=
GINKGO_NODES ?= 1
GINKGO_NODES ?= 4
GINKGO_TIMEOUT ?= 2h
GINKGO_POLL_PROGRESS_AFTER ?= 60m
GINKGO_POLL_PROGRESS_INTERVAL ?= 5m
E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/fixture.yaml
E2E_BMCS_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/bmcs-fixture.yaml
USE_EXISTING_CLUSTER ?= false
SKIP_RESOURCE_CLEANUP ?= false
GINKGO_NOCOLOR ?= false
Expand Down Expand Up @@ -118,7 +119,8 @@ test-e2e: $(GINKGO) ## Run the end-to-end tests
-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" \
$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) test/e2e -- \
-e2e.config="$(E2E_CONF_FILE)" -e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
-e2e.config="$(E2E_CONF_FILE)" -e2e.bmcsConfig="$(E2E_BMCS_CONF_FILE)" \
-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
-e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) -e2e.artifacts-folder="$(ARTIFACTS)"

## --------------------------------------
Expand Down
33 changes: 14 additions & 19 deletions hack/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -59,39 +59,20 @@ minikube start
# Load the BMO e2e image into it
minikube image load quay.io/metal3-io/baremetal-operator:e2e

# Create libvirt domain
VM_NAME="bmo-e2e-0"
export BOOT_MAC_ADDRESS="00:60:2f:31:81:01"

"${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${VM_NAME}" "${BOOT_MAC_ADDRESS}"

# This IP is defined by the network we created above.
IP_ADDRESS="192.168.222.1"

# These variables are used by the tests. They override variables in the config file.
# These are the VBMC defaults (used since we did not specify anything else for `vbmc add`).
export BMC_USER=admin
export BMC_PASSWORD=password

if [[ "${BMO_E2E_EMULATOR}" == "vbmc" ]]; then
# VBMC variables
VBMC_PORT="16230"
export BMC_ADDRESS="ipmi://${IP_ADDRESS}:${VBMC_PORT}"

# Start VBMC
docker run --name vbmc --network host -d \
-v /var/run/libvirt/libvirt-sock:/var/run/libvirt/libvirt-sock \
-v /var/run/libvirt/libvirt-sock-ro:/var/run/libvirt/libvirt-sock-ro \
quay.io/metal3-io/vbmc

# Add BMH VM to VBMC
"${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${VM_NAME}" "${VBMC_PORT}"

elif [[ "${BMO_E2E_EMULATOR}" == "sushy-tools" ]]; then
# Sushy-tools variables
SUSHY_EMULATOR_FILE="${REPO_ROOT}"/test/e2e/sushy-tools/sushy-emulator.conf
SUSHY_PORT="8000"
export BMC_ADDRESS="redfish+http://${IP_ADDRESS}:${SUSHY_PORT}/redfish/v1/Systems/${VM_NAME}"

# Start sushy-tools
docker run --name sushy-tools -d --network host \
Expand All @@ -105,6 +86,19 @@ else
exit 1
fi

export E2E_BMCS_CONF_FILE="${REPO_ROOT}/test/e2e/config/bmcs-${BMO_E2E_EMULATOR}.yaml"
"${REPO_ROOT}/hack/create_bmcs.sh" "${E2E_BMCS_CONF_FILE}"

# Set the number of ginkgo processes to the smaller number in
# the number of host CPUs and the number of BMCs
n_cpus=$(nproc)
n_vms=$(yq '. | length' "${E2E_BMCS_CONF_FILE}")
if [ "${n_cpus}" -lt "${n_vms}" ]; then
export GINKGO_NODES="${n_cpus}"
else
export GINKGO_NODES="${n_vms}"
fi

# Image server variables
CIRROS_VERSION="0.6.2"
IMAGE_FILE="cirros-${CIRROS_VERSION}-x86_64-disk.img"
Expand Down Expand Up @@ -153,6 +147,7 @@ echo "IRONIC_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_USERNAME}" "${IRONIC_PASSWOR
echo "INSPECTOR_HTPASSWD=$(htpasswd -n -b -B "${IRONIC_INSPECTOR_USERNAME}" \
"${IRONIC_INSPECTOR_PASSWORD}")" > "${IRONIC_OVERLAY}/ironic-inspector-htpasswd"


# We need to gather artifacts/logs before exiting also if there are errors
set +e

Expand Down
14 changes: 11 additions & 3 deletions hack/clean-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,19 @@ minikube delete
docker rm -f vbmc
docker rm -f image-server-e2e
docker rm -f sushy-tools
virsh -c qemu:///system destroy --domain bmo-e2e-0
virsh -c qemu:///system undefine --domain bmo-e2e-0 --nvram --remove-all-storage

virsh_vms=$(virsh list --name --all)

for vm in ${virsh_vms}; do
if [[ "${vm}" =~ "bmo-e2e-" ]]; then
virsh -c qemu:///system destroy --domain "${vm}"
virsh -c qemu:///system undefine --domain "${vm}" --nvram --remove-all-storage
fi
done

virsh -c qemu:///system net-destroy baremetal-e2e
virsh -c qemu:///system net-undefine baremetal-e2e

rm -rf "${REPO_ROOT}/test/e2e/_artifacts"
rm -rf "${REPO_ROOT}/artifacts.tar.gz"
rm -rf "${REPO_ROOT}"/artifacts-*
rm -rf "${REPO_ROOT}/test/e2e/images"
37 changes: 37 additions & 0 deletions hack/create_bmcs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
#
# This script reads BMC information in a config file and prepare VMs
# whose info match those config
#
set -eux

REPO_ROOT=$(realpath "$(dirname "${BASH_SOURCE[0]}")/..")
cd "${REPO_ROOT}" || exit 1

CONFIG_FILE=$1
NETWORK=${2:-"baremetal-e2e"}

readarray -t BMCs < <(yq e -o=j -I=0 '.[]' "${CONFIG_FILE}")

# N_BMCS=$(yq '.bmcs | length' "${CONFIG_FILE}")
# echo ${N_BMCS}
for bmc in "${BMCs[@]}"; do
address=$(echo "${bmc}" | jq -r '.address')
bootMacAddress=$(echo "${bmc}" | jq -r '.bootMacAddress')
hostName=$(echo "${bmc}" | jq -r '.hostName')
ipAddress=$(echo "${bmc}" | jq -r '.ipAddress')

# Create VM
"${REPO_ROOT}/tools/bmh_test/create_vm.sh" "${hostName}" "${bootMacAddress}"

# Add the the VM to the network host list
virsh net-update "${NETWORK}" add-last ip-dhcp-host \
"<host mac='${bootMacAddress}' name='${hostName}' ip='${ipAddress}' />" \
--live --config --parent-index 0

# Add BMH VM to VBMC
if [[ "${address}" =~ "ipmi://" ]]; then
vbmc_port="${address##*:}"
"${REPO_ROOT}/tools/bmh_test/vm2vbmc.sh" "${hostName}" "${vbmc_port}"
fi
done
1 change: 0 additions & 1 deletion hack/e2e/net.xml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
<dhcp>
<range start='192.168.222.2' end='192.168.222.199'/>
<host mac='52:54:00:6c:3c:01' name='minikube' ip='192.168.222.199'/>
<host mac='00:60:2f:31:81:01' name='bmo-e2e-0' ip='192.168.222.122'/>
<bootp file='http://192.168.222.199:6180/boot.ipxe'/>
</dhcp>
</ip>
Expand Down
26 changes: 10 additions & 16 deletions test/e2e/basic_ops_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,18 @@ import (

var _ = Describe("basic", func() {
var (
specName = "basic-ops"
secretName = "bmc-credentials"
namespace *corev1.Namespace
cancelWatches context.CancelFunc
bmcUser string
bmcPassword string
bmcAddress string
bootMacAddress string
specName = "basic-ops"
secretName = "bmc-credentials"
namespace *corev1.Namespace
cancelWatches context.CancelFunc
bmc BMC
)
const (
rebootAnnotation = "reboot.metal3.io"
poweroffAnnotation = "reboot.metal3.io/poweroff"
)
BeforeEach(func() {
bmcUser = e2eConfig.GetVariable("BMC_USER")
bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
bmc = (*bmcs)[GinkgoParallelProcess()-1]

namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
Creator: clusterProxy.GetClient(),
Expand All @@ -49,8 +43,8 @@ var _ = Describe("basic", func() {
It("should control power cycle of BMH though annotations", func() {
By("creating a secret with BMH credentials")
bmcCredentialsData := map[string]string{
"username": bmcUser,
"password": bmcPassword,
"username": bmc.User,
"password": bmc.Password,
}
CreateSecret(ctx, clusterProxy.GetClient(), namespace.Name, secretName, bmcCredentialsData)

Expand All @@ -66,11 +60,11 @@ var _ = Describe("basic", func() {
Spec: metal3api.BareMetalHostSpec{
Online: true,
BMC: metal3api.BMCDetails{
Address: bmcAddress,
Address: bmc.Address,
CredentialsName: "bmc-credentials",
},
BootMode: metal3api.Legacy,
BootMACAddress: bootMacAddress,
BootMACAddress: bmc.BootMacAddress,
},
}
err := clusterProxy.GetClient().Create(ctx, &bmh)
Expand Down
27 changes: 27 additions & 0 deletions test/e2e/bmc.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package e2e

import (
"os"

. "github.com/onsi/gomega"
"gopkg.in/yaml.v2"
)

// BMC defines a BMH to use in the tests
type BMC struct {
User string `yaml:"user,omitempty"`
Password string `yaml:"password,omitempty"`
Address string `yaml:"address,omitempty"`
BootMacAddress string `yaml:"bootMacAddress,omitempty"`
HostName string `yaml:"hostName,omitempty"`
IpAddress string `yaml:"ipAddress,omitempty"`
SSHPort string `yaml:"sshPort,omitempty"`
}

func LoadBMCConfig(configPath string) *[]BMC {
configData, err := os.ReadFile(configPath) //#nosec
Expect(err).ToNot(HaveOccurred(), "Failed to read the bmcs config file")
var bmcs []BMC
Expect(yaml.Unmarshal(configData, &bmcs)).To(Succeed())
return &bmcs
}
7 changes: 2 additions & 5 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -391,13 +391,10 @@ echo "%s" >> /root/.ssh/authorized_keys`, sshPubKeyData, sshPubKeyData)
// PerformSSHBootCheck performs an SSH check to verify the node's boot source.
// The `expectedBootMode` parameter should be "disk" or "memory".
// The `auth` parameter is an ssh.AuthMethod for authentication.
func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod) {
ip := e2eConfig.GetVariable("IP_BMO_E2E_0")
sshPort := e2eConfig.GetVariable("SSH_PORT")
address := fmt.Sprintf("%s:%s", ip, sshPort)
func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.AuthMethod, sshAddress string) {
user := e2eConfig.GetVariable("CIRROS_USERNAME")

client := EstablishSSHConnection(e2eConfig, auth, user, address)
client := EstablishSSHConnection(e2eConfig, auth, user, sshAddress)
defer func() {
if client != nil {
client.Close()
Expand Down
30 changes: 30 additions & 0 deletions test/e2e/config/bmcs-fixture.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# For fixture config, only the `hostName` field matters. The number of BMCs,
# however, needs to be equal or larger than `GINKGO_NODES`.
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
bootMacAddress: "00:60:2f:31:81:01"
hostName: ""
ipAddress: "192.168.222.122"
sshPort: "22"
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1"
bootMacAddress: "00:60:2f:31:81:02"
hostName: ""
ipAddress: "192.168.222.123"
sshPort: "22"
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-2"
bootMacAddress: "00:60:2f:31:81:01"
hostName: ""
ipAddress: "192.168.222.124"
sshPort: "22"
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-3"
bootMacAddress: "00:60:2f:31:81:02"
hostName: ""
ipAddress: "192.168.222.125"
sshPort: "22"
21 changes: 21 additions & 0 deletions test/e2e/config/bmcs-sushy-tools.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
bootMacAddress: "00:60:2f:31:81:01"
hostName: "bmo-e2e-0"
ipAddress: "192.168.222.122"
sshPort: "22"
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-1"
bootMacAddress: "00:60:2f:31:81:02"
hostName: "bmo-e2e-1"
ipAddress: "192.168.222.123"
sshPort: "22"
- user: admin
password: password
address: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-2"
bootMacAddress: "00:60:2f:31:81:03"
hostName: "bmo-e2e-2"
ipAddress: "192.168.222.124"
sshPort: "22"
21 changes: 21 additions & 0 deletions test/e2e/config/bmcs-vbmc.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
- user: admin
password: password
address: "ipmi://192.168.222.1:16230"
bootMacAddress: "00:60:2f:31:81:01"
hostName: "bmo-e2e-0"
ipAddress: "192.168.222.122"
sshPort: "22"
- user: admin
password: password
address: "ipmi://192.168.222.1:16231"
bootMacAddress: "00:60:2f:31:81:02"
hostName: "bmo-e2e-1"
ipAddress: "192.168.222.123"
sshPort: "22"
- user: admin
password: password
address: "ipmi://192.168.222.1:16232"
bootMacAddress: "00:60:2f:31:81:03"
hostName: "bmo-e2e-2"
ipAddress: "192.168.222.124"
sshPort: "22"
11 changes: 2 additions & 9 deletions test/e2e/config/fixture.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,6 @@ variables:
UPGRADE_DEPLOY_CERT_MANAGER: "true"
UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.4"

BMC_ADDRESS: ipmi://192.168.222.1:16230
BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
EXPECTED_HOST_NAME: ""
# Test credentials. The tests will create a BMH with these.
# For the fixture provider it does not matter much what we put here.
BMC_USER: admin
BMC_PASSWORD: password
IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
CERT_MANAGER_VERSION: "v1.13.1"
Expand All @@ -45,11 +38,11 @@ intervals:
inspection/wait-available: ["5s", "1ms"]
external-inspection/wait-available: ["5s", "1ms"]
default/wait-deployment: ["5m", "1s"]
default/wait-namespace-deleted: ["20s", "1s"]
default/wait-namespace-deleted: ["30s", "1s"]
ironic/wait-deployment: ["10m", "2s"]
default/wait-registration: ["5s", "5ms"]
default/wait-provisioning: ["30s", "10ms"]
default/wait-provisioned: ["1m", "1s"]
default/wait-provisioned: ["2m", "1s"]
default/wait-deprovisioning: ["5s", "10ms"]
default/wait-available: ["2m", "1s"]
default/wait-deleting: ["5s", "10ms"]
Expand Down
14 changes: 1 addition & 13 deletions test/e2e/config/ironic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,29 +22,17 @@ variables:
DEPLOY_CERT_MANAGER: "true"
BMO_KUSTOMIZATION: "../../config/overlays/e2e"
IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"

# These settings are for the separate cluster that are used for upgrade tests
UPGRADE_DEPLOY_IRONIC: "true"
UPGRADE_DEPLOY_BMO: "true"
UPGRADE_DEPLOY_CERT_MANAGER: "true"
UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.4"

# Test credentials. The tests will create a BMH with these.
BMC_USER: admin
BMC_PASSWORD: password
# During development, select the preferred BMC interface when running tests from the command line:
# Note: These settings are only applicable for command line executions, not when running via the ci-e2e.sh script.
# Uncomment the desired BMC_ADDRESS line:
# BMC_ADDRESS: "ipmi://192.168.222.1:16230"
BMC_ADDRESS: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
EXPECTED_HOST_NAME: "bmo-e2e-0"

IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
CERT_MANAGER_VERSION: "v1.13.1"
SSH_CHECK_PROVISIONED: "true"
IP_BMO_E2E_0: "192.168.222.122"
SSH_PORT: "22"
CIRROS_USERNAME: "cirros"
CIRROS_PASSWORD: "gocubsgo"
SSH_PRIV_KEY: "./images/ssh_testkey"
Expand Down

0 comments on commit f57b2ba

Please sign in to comment.