Skip to content

Commit

Permalink
Add e2e basic operation test
Browse files Browse the repository at this point in the history
  • Loading branch information
mohammed boukhalfa committed Oct 12, 2023
1 parent 7c007b6 commit f94b916
Show file tree
Hide file tree
Showing 7 changed files with 266 additions and 81 deletions.
8 changes: 4 additions & 4 deletions hack/ci-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,16 @@ docker run --name vbmc --network host -d \
quay.io/metal3-io/vbmc

# Create libvirt domain
NAME="bmo-e2e-0"
export VM_NAME="bmo-e2e-0"
BOOT_MAC_ADDRESS="00:60:2f:31:81:01"
VBMC_PORT="16230"
virt-install --connect qemu:///system -n "${NAME}" --description "Virtualized BareMetalHost" --osinfo=ubuntu-lts-latest \
virt-install --connect qemu:///system -n "${VM_NAME}" --description "Virtualized BareMetalHost" --osinfo=ubuntu-lts-latest \
--ram=4096 --vcpus=2 --disk size=20 --graphics=none --console pty --serial pty --pxe \
--network network=baremetal-e2e,mac="${BOOT_MAC_ADDRESS}" --noautoconsole

# Add BMH VM to VBMC
docker exec vbmc vbmc add "${NAME}" --port "${VBMC_PORT}"
docker exec vbmc vbmc start "${NAME}"
docker exec vbmc vbmc add "${VM_NAME}" --port "${VBMC_PORT}"
docker exec vbmc vbmc start "${VM_NAME}"
docker exec vbmc vbmc list

# These variables are used by the tests. They override variables in the config file.
Expand Down
1 change: 1 addition & 0 deletions hack/clean-e2e.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ docker stop vbmc
docker rm vbmc
virsh -c qemu:///system destroy --domain bmo-e2e-0
virsh -c qemu:///system undefine --domain bmo-e2e-0 --remove-all-storage
virsh -c qemu:///system net-destroy baremetal-e2e
virsh -c qemu:///system net-undefine baremetal-e2e

rm -rf "${REPO_ROOT}/test/e2e/_artifacts"
123 changes: 123 additions & 0 deletions test/e2e/basic_ops_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package e2e

import (
"context"
"fmt"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
"sigs.k8s.io/cluster-api/test/framework"
"sigs.k8s.io/cluster-api/util"

capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e"

metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
)

var _ = Describe("basic", func() {
var (
specName = "basic-ops"
namespace *corev1.Namespace
cancelWatches context.CancelFunc
bmcUser string
bmcPassword string
bmcAddress string
bootMacAddress string
vmName string
)
const (
rebootAnnotation = "reboot.metal3.io"
poweroffAnnotation = "reboot.metal3.io/poweroff"
)
BeforeEach(func() {
bmcUser = e2eConfig.GetVariable("BMC_USER")
bmcPassword = e2eConfig.GetVariable("BMC_PASSWORD")
bmcAddress = e2eConfig.GetVariable("BMC_ADDRESS")
bootMacAddress = e2eConfig.GetVariable("BOOT_MAC_ADDRESS")
vmName = e2eConfig.GetVariable("VM_NAME")

namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
Creator: clusterProxy.GetClient(),
ClientSet: clusterProxy.GetClientSet(),
Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
LogFolder: artifactFolder,
})
})

It("should power cycle a newly created BMH", func() {
By("creating a secret with BMH credentials")
bmcCredentials := corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: "bmc-credentials",
Namespace: namespace.Name,
},
StringData: map[string]string{
"username": bmcUser,
"password": bmcPassword,
},
}
err := clusterProxy.GetClient().Create(ctx, &bmcCredentials)
Expect(err).NotTo(HaveOccurred())

By("creating a BMH")
bmh := metal3api.BareMetalHost{
ObjectMeta: metav1.ObjectMeta{
Name: specName + "-powercycle",
Namespace: namespace.Name,
Annotations: map[string]string{
"inspect.metal3.io": "disabled",
},
},
Spec: metal3api.BareMetalHostSpec{
Online: true,
BMC: metal3api.BMCDetails{
Address: bmcAddress,
CredentialsName: "bmc-credentials",
},
BootMode: metal3api.Legacy,
BootMACAddress: bootMacAddress,
},
}
err = clusterProxy.GetClient().Create(ctx, &bmh)
Expect(err).NotTo(HaveOccurred())

By("waiting for the BMH to be in registering state")
WaitForBmhInState(ctx, WaitForBmhInStateInput{
Client: clusterProxy.GetClient(),
Bmh: bmh,
State: metal3api.StateRegistering,
}, e2eConfig.GetIntervals("inspection", "wait-registering")...)

By("waiting for the BMH to become available")
WaitForBmhInState(ctx, WaitForBmhInStateInput{
Client: clusterProxy.GetClient(),
Bmh: bmh,
State: metal3api.StateAvailable,
}, e2eConfig.GetIntervals("inspection", "wait-available")...)

By("testing the reboot BMH")
capm3_e2e.AnnotateBmh(ctx, clusterProxy.GetClient(), bmh, rebootAnnotation, pointer.String("{\"force\": true}"))
waitForVmsState([]string{vmName}, shutoff, specName, e2eConfig.GetIntervals(specName, "wait-vm-state")...)
waitForVmsState([]string{vmName}, running, specName, e2eConfig.GetIntervals(specName, "wait-vm-state")...)

By("checking the power off BMH")
capm3_e2e.AnnotateBmh(ctx, clusterProxy.GetClient(), bmh, poweroffAnnotation, pointer.String("{\"force\": true}"))
waitForVmsState([]string{vmName}, shutoff, specName, e2eConfig.GetIntervals(specName, "wait-vm-state")...)

// power on
By("checking the power on BMH")
capm3_e2e.AnnotateBmh(ctx, clusterProxy.GetClient(), bmh, poweroffAnnotation, nil)
waitForVmsState([]string{vmName}, running, specName, e2eConfig.GetIntervals(specName, "wait-vm-state")...)
})

AfterEach(func() {
framework.DeleteNamespace(ctx, framework.DeleteNamespaceInput{
Deleter: clusterProxy.GetClient(),
Name: namespace.Name,
})
cancelWatches()
})
})
49 changes: 47 additions & 2 deletions test/e2e/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,17 +4,19 @@ import (
"context"
"fmt"
"os"
"os/exec"
"runtime"
"strings"
"time"

"github.com/pkg/errors"
"gopkg.in/yaml.v2"

metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"

capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e"
. "github.com/onsi/gomega"

metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"

"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/cluster-api/test/framework/clusterctl"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand All @@ -33,6 +35,15 @@ const (
TryLoadImage LoadImageBehavior = "tryLoad"
)

type vmState string

const (
running vmState = "running"
paused vmState = "paused"
shutoff vmState = "shutoff"
other vmState = "other"
)

// Config defines the configuration of an e2e test environment.
type Config struct {
// Images is a list of container images to load into the Kind cluster.
Expand Down Expand Up @@ -153,3 +164,37 @@ func WaitForBmhInState(ctx context.Context, input WaitForBmhInStateInput, interv
g.Expect(bmh.Status.Provisioning.State).To(Equal(input.State))
}, intervals...).Should(Succeed())
}

func waitForVmsState(vmNames []string, state vmState, _ string, interval ...interface{}) {
capm3_e2e.Byf("Waiting for VMs %v to become '%s'", vmNames, state)
Eventually(func() []string {
return listVms(state)
}, interval...).Should(ContainElements(vmNames))
}
func listVms(state vmState) []string {
var cmd *exec.Cmd // gosec Subprocess launched with variable
switch state {
case running:
cmd = exec.Command("sudo", "virsh", "list", "--name", "--state-running")
case shutoff:
cmd = exec.Command("sudo", "virsh", "list", "--name", "--state-shutoff")
case paused:
cmd = exec.Command("sudo", "virsh", "list", "--name", "--state-paused")
case other:
cmd = exec.Command("sudo", "virsh", "list", "--name", "--state-other")
}

result, err := cmd.Output()
Expect(err).NotTo(HaveOccurred())

lines := strings.Split(string(result), "\n")
// virsh may return some empty lines which need to be removed
i := 0
for _, line := range lines {
if line != "" {
lines[i] = line
i++
}
}
return lines[:i]
}
3 changes: 2 additions & 1 deletion test/e2e/config/ironic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ variables:
BMC_PASSWORD: password
BMC_ADDRESS: ipmi://192.168.222.1:16230
BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"

VM_NAME: "bmo-e2e-0"
intervals:
inspection/wait-unmanaged: ["1m", "5s"]
inspection/wait-registering: ["1m", "5s"]
Expand All @@ -26,3 +26,4 @@ intervals:
inspection/wait-available: ["10m", "10s"]
default/wait-deployment: ["5m", "1s"]
ironic/wait-deployment: ["10m", "2s"]
default/wait-vm-state: ["20m", "100ms"]
56 changes: 31 additions & 25 deletions test/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,41 +3,45 @@ module github.com/metal3-io/baremetal-operator/test
go 1.20

require (
github.com/metal3-io/baremetal-operator/apis v0.3.1
github.com/onsi/ginkgo/v2 v2.11.0
github.com/metal3-io/baremetal-operator/apis v0.4.0
github.com/metal3-io/cluster-api-provider-metal3/test v1.5.1
github.com/onsi/ginkgo/v2 v2.12.0
github.com/onsi/gomega v1.27.10
github.com/pkg/errors v0.9.1
gopkg.in/yaml.v2 v2.4.0
k8s.io/api v0.27.2
k8s.io/apimachinery v0.27.2
k8s.io/api v0.27.5
k8s.io/apimachinery v0.27.5
k8s.io/klog/v2 v2.90.1
sigs.k8s.io/cluster-api v1.5.0
sigs.k8s.io/cluster-api/test v1.5.0
sigs.k8s.io/controller-runtime v0.15.0
k8s.io/utils v0.0.0-20230209194617-a36077c30491
sigs.k8s.io/cluster-api v1.5.2
sigs.k8s.io/cluster-api/test v1.5.1
sigs.k8s.io/controller-runtime v0.15.2
)

replace github.com/metal3-io/cluster-api-provider-metal3/api => github.com/metal3-io/cluster-api-provider-metal3/api v1.5.1

require (
github.com/BurntSushi/toml v1.0.0 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver/v3 v3.2.0 // indirect
github.com/Masterminds/sprig/v3 v3.2.3 // indirect
github.com/Microsoft/go-winio v0.5.0 // indirect
github.com/Microsoft/go-winio v0.6.0 // indirect
github.com/adrg/xdg v0.4.0 // indirect
github.com/alessio/shellescape v1.4.1 // indirect
github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/blang/semver v3.5.1+incompatible // indirect
github.com/blang/semver/v4 v4.0.0 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/coredns/caddy v1.1.0 // indirect
github.com/coredns/corefile-migration v1.0.20 // indirect
github.com/coredns/caddy v1.1.1 // indirect
github.com/coredns/corefile-migration v1.0.21 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/distribution v2.8.2+incompatible // indirect
github.com/docker/docker v24.0.5+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/docker/go-units v0.4.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/drone/envsubst/v2 v2.0.0-20210730161058-179042472c46 // indirect
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
Expand All @@ -52,17 +56,17 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/cel-go v0.12.6 // indirect
github.com/google/cel-go v0.13.0 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/go-github/v48 v48.2.0 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/google/uuid v1.3.1 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/huandu/xstrings v1.3.3 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
Expand All @@ -71,7 +75,9 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.2.0 // indirect
github.com/metal3-io/baremetal-operator/pkg/hardwareutils v0.4.0 // indirect
github.com/metal3-io/cluster-api-provider-metal3/api v0.0.0 // indirect
github.com/metal3-io/ip-address-manager/api v1.5.1 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
Expand All @@ -96,29 +102,29 @@ require (
github.com/stoewer/go-strcase v1.2.0 // indirect
github.com/subosito/gotenv v1.4.2 // indirect
github.com/valyala/fastjson v1.6.4 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/net v0.12.0 // indirect
golang.org/x/crypto v0.12.0 // indirect
golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.14.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/term v0.11.0 // indirect
golang.org/x/text v0.12.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.9.3 // indirect
golang.org/x/tools v0.12.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
gotest.tools/v3 v3.5.0 // indirect
k8s.io/apiextensions-apiserver v0.27.2 // indirect
k8s.io/apiserver v0.27.2 // indirect
k8s.io/client-go v0.27.2 // indirect
k8s.io/client-go v0.27.5 // indirect
k8s.io/cluster-bootstrap v0.27.2 // indirect
k8s.io/component-base v0.27.2 // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/kind v0.20.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
Expand Down

0 comments on commit f94b916

Please sign in to comment.