Add bmo upgrade test

Signed-off-by: Huy Mai <huy.mai@est.tech>

.github/workflows/e2e-fixture-test.yml

@@ -30,6 +30,6 @@ jobs:
 
     - name: Set Up Environment and Run BMO e2e Tests
       env:
-        E2E_CONF_FILE: ${GITHUB_WORKSPACE}/test/e2e/config/fixture.yaml
+        E2E_CONF_FILE: ${{ github.workspace }}/test/e2e/config/fixture.yaml
         USE_EXISTING_CLUSTER: "false"
       run: make test-e2e

.gitignore

@@ -3,7 +3,7 @@ bin
 build/_output
 build/_test
 _artifacts
-artifacts.tar.gz
+artifacts*.tar.gz
 test/e2e/images
 
 # Created by https://www.gitignore.io/api/go

Makefile

@@ -50,8 +50,8 @@ GINKGO_NODES ?= 1
 GINKGO_TIMEOUT ?= 2h
 GINKGO_POLL_PROGRESS_AFTER ?= 60m
 GINKGO_POLL_PROGRESS_INTERVAL ?= 5m
-E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/ironic.yaml
-USE_EXISTING_CLUSTER ?= true
+E2E_CONF_FILE ?= $(ROOT_DIR)/test/e2e/config/fixture.yaml
+USE_EXISTING_CLUSTER ?= false
 SKIP_RESOURCE_CLEANUP ?= false
 GINKGO_NOCOLOR ?= false
 

config/overlays/e2e-release-0.4/ironic.env

@@ -0,0 +1,10 @@
+HTTP_PORT=6180
+PROVISIONING_IP=192.168.222.199
+DEPLOY_KERNEL_URL=http://192.168.222.199:6180/images/ironic-python-agent.kernel
+DEPLOY_RAMDISK_URL=http://192.168.222.199:6180/images/ironic-python-agent.initramfs
+IRONIC_ENDPOINT=https://192.168.222.199:6385/v1/
+IRONIC_INSPECTOR_ENDPOINT=https://192.168.222.199:5050/v1/
+CACHEURL=http://192.168.222.199/images
+IRONIC_FAST_TRACK=true
+IRONIC_KERNEL_PARAMS=console=ttyS0
+IRONIC_INSPECTOR_VLAN_INTERFACES=all

config/overlays/e2e-release-0.4/kustomization.yaml

@@ -0,0 +1,45 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: baremetal-operator-system
+resources:
+  - https://github.com/metal3-io/baremetal-operator/config/tls?ref=release-0.4
+
+components:
+  - ../../components/basic-auth
+
+configMapGenerator:
+- name: ironic
+  behavior: merge
+  envs:
+  - ironic.env
+
+patches:
+- patch: |
+    # Don't try to pull again the pre-loaded image
+    - op: replace
+      path: /spec/template/spec/containers/0/imagePullPolicy
+      value: IfNotPresent
+  target:
+    kind: Deployment
+    name: controller-manager
+
+images:
+- name: quay.io/metal3-io/baremetal-operator
+  newTag: release-0.4
+
+# We cannot use suffix hashes since the kustomizations we build on
+# cannot be aware of what suffixes we add.
+generatorOptions:
+  disableNameSuffixHash: true
+
+# NOTE: These credentials are generated automatically in hack/ci-e2e.sh
+secretGenerator:
+  - name: ironic-credentials
+    files:
+    - username=ironic-username
+    - password=ironic-password
+  - name: ironic-inspector-credentials
+    files:
+    - username=ironic-inspector-username
+    - password=ironic-inspector-password
+

config/overlays/fixture-release-0.4/kustomization.yaml

@@ -0,0 +1,24 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+- https://github.com/metal3-io/baremetal-operator/config/namespace?ref=release-0.4
+- https://github.com/metal3-io/baremetal-operator/config/default?ref=release-0.4
+
+patches:
+- patch: |
+    # Enable test mode (fixture provider instead of ironic)
+    - op: add
+      path: /spec/template/spec/containers/0/args/-
+      value: --test-mode
+    # Don't try to pull again the pre-loaded image
+    - op: replace
+      path: /spec/template/spec/containers/0/imagePullPolicy
+      value: IfNotPresent
+  target:
+    kind: Deployment
+    name: controller-manager
+
+images:
+- name: quay.io/metal3-io/baremetal-operator
+  newTag: release-0.4

hack/ci-e2e.sh

@@ -17,6 +17,21 @@ cd "${REPO_ROOT}" || exit 1
 # BMO_E2E_EMULATOR can be set to either "vbmc" or "sushy-tools"
 BMO_E2E_EMULATOR=${BMO_E2E_EMULATOR:-"sushy-tools"}
 
+case "${GINKGO_FOCUS:-}" in
+  *upgrade*)
+    echo "'upgrade' found in env var GINKGO_FOCUS. Will ONLY run upgrade tests."
+    export GINKGO_FOCUS="upgrade"
+    export DEPLOY_IRONIC="false"
+    export DEPLOY_BMO="false"
+    export DEPLOY_CERT_MANAGER="false"
+    ;;
+  *)
+    echo "'upgrade' not found in env var GINKGO_FOCUS. Will NOT run upgrade tests."
+    export GINKGO_SKIP="upgrade ${GINKGO_SKIP:-}"
+    ;;
+esac
+export USE_EXISTING_CLUSTER="true"
+
 # Ensure requirements are installed
 "${REPO_ROOT}/hack/e2e/ensure_go.sh"
 export PATH="${PATH}:/usr/local/go/bin"
@@ -142,6 +157,14 @@ echo "${IRONIC_PASSWORD}" > "${BMO_OVERLAY}/ironic-password"
 echo "${IRONIC_INSPECTOR_USERNAME}" > "${BMO_OVERLAY}/ironic-inspector-username"
 echo "${IRONIC_INSPECTOR_PASSWORD}" > "${BMO_OVERLAY}/ironic-inspector-password"
 
+if [[ "${GINKGO_FOCUS:-}" == "upgrade" ]]; then
+  BMO_UPGRADE_FROM_OVERLAY="${REPO_ROOT}/config/overlays/e2e-release-0.4"
+  echo "${IRONIC_USERNAME}" > "${BMO_UPGRADE_FROM_OVERLAY}/ironic-username"
+  echo "${IRONIC_PASSWORD}" > "${BMO_UPGRADE_FROM_OVERLAY}/ironic-password"
+  echo "${IRONIC_INSPECTOR_USERNAME}" > "${BMO_UPGRADE_FROM_OVERLAY}/ironic-inspector-username"
+  echo "${IRONIC_INSPECTOR_PASSWORD}" > "${BMO_UPGRADE_FROM_OVERLAY}/ironic-inspector-password"
+fi
+
 envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-auth-config-tpl" > \
   "${IRONIC_OVERLAY}/ironic-auth-config"
 envsubst < "${REPO_ROOT}/ironic-deployment/components/basic-auth/ironic-inspector-auth-config-tpl" > \

test/e2e/README.md

@@ -11,15 +11,28 @@ in `config`.
 In the CI system we set up a pre-configured minikube cluster that is then used
 for the tests, as seen in [ci-e2e.sh](../../hack/ci-e2e.sh). This allows us to
 have control over the network and have a static configuration for Ironic and the
-Baremetal Operator. The script also creates a VM and configures VBMC to be used
-as BareMetalHost and BMC respectively.
+Baremetal Operator. The script creates a VM to represent a BareMetalHost,
+and configures either VBMC or sushy-tools to be used as BMC. This script can be
+configured to run either [E2E upgrade tests](upgrade_test.go), or the "normal"
+E2E tests set (all tests except the upgrade ones) by exporting the `GINKGO_FOCUS`
+env var. If "upgrade" is found as a part of `GINKGO_FOCUS`, ONLY the upgrade tests
+will be run, otherwise the upgrade tests are skipped (other tests in `GINKGO_FOCUS`
+or `GINKGO_SKIP` will be kept as-is).
+
+Here is how to run the E2E upgrade tests:
+
+```bash
+export GINKGO_FOCUS="upgrade"
+./hack/ci-e2e.sh
+```
 
 It is also possible to run the tests with the fixture provider instead of
-Ironic, but please note that it is quite questionable to call this configuration
-"end to end". This is how to run the e2e test suite with the fixture provider:
+Ironic. Without any changes, the whole suite (including upgrade tests) will be run.
+Please note, however, that it is quite questionable to call this configuration
+"end to end".
+
+Fixture provider is configured to be the default one. This is how to run the e2e test suite with it:
 
 ```bash
-export E2E_CONF_FILE=${REPO_ROOT}/test/e2e/config/fixture.yaml
-export USE_EXISTING_CLUSTER="false"
 make test-e2e
 ```

test/e2e/common.go

@@ -505,3 +505,20 @@ func BuildAndApplyKustomize(ctx context.Context, input *BuildAndApplyKustomizeIn
 	}
 	return nil
 }
+
+func DeploymentRolledOut(ctx context.Context, clusterProxy framework.ClusterProxy, name string, namespace string, desiredGeneration int64) bool {
+	clientSet := clusterProxy.GetClientSet()
+	deploy, err := clientSet.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{})
+	Expect(err).To(BeNil())
+	if deploy != nil {
+		// When the number of replicas is equal to the number of available and updated
+		// replicas, we know that only "new" pods are running. When we also
+		// have the desired number of replicas and a new enough generation, we
+		// know that the rollout is complete.
+		return (deploy.Status.UpdatedReplicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.AvailableReplicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.Replicas == *deploy.Spec.Replicas) &&
+			(deploy.Status.ObservedGeneration >= desiredGeneration)
+	}
+	return false
+}

test/e2e/config/fixture.yaml

@@ -3,6 +3,8 @@ images:
 - name: quay.io/metal3-io/baremetal-operator:e2e
   loadBehavior: tryLoad
 # Save some time and network by using cached images if available
+- name: quay.io/metal3-io/baremetal-operator:release-0.4
+  loadBehavior: tryLoad
 - name: quay.io/jetstack/cert-manager-cainjector:v1.13.1
   loadBehavior: tryLoad
 - name: quay.io/jetstack/cert-manager-webhook:v1.13.1
@@ -11,17 +13,25 @@ images:
   loadBehavior: tryLoad
 
 variables:
+  # These settings are for the "default" cluster, which is used for all tests except upgrade ones
   DEPLOY_IRONIC: "false"
   DEPLOY_BMO: "true"
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/fixture"
+
+  # These settings are for the separate cluster that are used for upgrade tests
+  UPGRADE_DEPLOY_IRONIC: "false"
+  UPGRADE_DEPLOY_BMO: "true"
+  UPGRADE_DEPLOY_CERT_MANAGER: "true"
+  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/fixture-release-0.4"
+
+  BMC_ADDRESS: ipmi://192.168.222.1:16230
+  BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
   EXPECTED_HOST_NAME: ""
   # Test credentials. The tests will create a BMH with these.
   # For the fixture provider it does not matter much what we put here.
   BMC_USER: admin
   BMC_PASSWORD: password
-  BMC_ADDRESS: ipmi://192.168.222.1:16230
-  BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"
@@ -38,10 +48,10 @@ intervals:
   default/wait-namespace-deleted: ["20s", "1s"]
   ironic/wait-deployment: ["10m", "2s"]
   default/wait-registration: ["5s", "5ms"]
-  default/wait-provisioning: ["5s", "10ms"]
-  default/wait-provisioned: ["20s", "1s"]
+  default/wait-provisioning: ["30s", "10ms"]
+  default/wait-provisioned: ["1m", "10s"]
   default/wait-deprovisioning: ["5s", "10ms"]
-  default/wait-available: ["20s", "1s"]
+  default/wait-available: ["2m", "1s"]
   default/wait-deleting: ["5s", "10ms"]
   default/wait-deleted: ["5s", "10ms"]
   default/wait-secret-deletion: ["5s", "10ms"]

test/e2e/config/ironic.yaml

@@ -5,6 +5,8 @@ images:
 - name: quay.io/metal3-io/ironic:e2e
   loadBehavior: tryLoad
 # Save some time and network by using cached images if available
+- name: quay.io/metal3-io/baremetal-operator:release-0.4
+  loadBehavior: tryLoad
 - name: quay.io/jetstack/cert-manager-cainjector:v1.13.1
   loadBehavior: tryLoad
 - name: quay.io/jetstack/cert-manager-webhook:v1.13.1
@@ -14,12 +16,18 @@ images:
 
 # These variables can be overridden with environment variables.
 variables:
+  # These settings are for the "default" cluster, which is used for all tests except upgrade ones
   DEPLOY_IRONIC: "true"
   DEPLOY_BMO: "true"
   DEPLOY_CERT_MANAGER: "true"
   BMO_KUSTOMIZATION: "../../config/overlays/e2e"
   IRONIC_KUSTOMIZATION: "../../ironic-deployment/overlays/e2e"
-  EXPECTED_HOST_NAME: "bmo-e2e-0"
+  # These settings are for the separate cluster that are used for upgrade tests
+  UPGRADE_DEPLOY_IRONIC: "true"
+  UPGRADE_DEPLOY_BMO: "true"
+  UPGRADE_DEPLOY_CERT_MANAGER: "true"
+  UPGRADE_BMO_KUSTOMIZATION_FROM: "../../config/overlays/e2e-release-0.4"
+
   # Test credentials. The tests will create a BMH with these.
   BMC_USER: admin
   BMC_PASSWORD: password
@@ -29,6 +37,8 @@ variables:
   # BMC_ADDRESS: "ipmi://192.168.222.1:16230"
   BMC_ADDRESS: "redfish+http://192.168.222.1:8000/redfish/v1/Systems/bmo-e2e-0"
   BOOT_MAC_ADDRESS: "00:60:2f:31:81:01"
+  EXPECTED_HOST_NAME: "bmo-e2e-0"
+
   IMAGE_URL: "http://192.168.222.1/cirros-0.6.2-x86_64-disk.img"
   IMAGE_CHECKSUM: "c8fc807773e5354afe61636071771906"
   CERT_MANAGER_VERSION: "v1.13.1"