[release-4.8] WINC-607: Add support for platform=none in e2e test suite

Makefile

@@ -179,6 +179,10 @@ unit:
 run-ci-e2e-test:
 	hack/run-ci-e2e-test.sh -t basic
 
+.PHONY: run-ci-e2e-byoh-test
+run-ci-e2e-byoh-test:
+	hack/run-ci-e2e-test.sh -t basic -m 0
+
 .PHONY: run-ci-e2e-upgrade-test
 run-ci-e2e-upgrade-test:
 	hack/run-ci-e2e-test.sh -t upgrade

docs/HACKING.md

@@ -93,6 +93,38 @@ hack/run-ci-e2e-test.sh -s -m 2 -c 1
 ```
 Please note that you do not need to run `hack/olm.sh run` before `hack/run-ci-e2e-test.sh`.
 
+#### Running e2e tests on platform-agnostic infrastructure
+
+To run the WMCO e2e tests on a bare metal or other platform-agnostic infrastructure (platform=none), where there
+is no cloud provider specification, the desired number of Windows instances must be provisioned beforehand, and the
+instance(s) information must be provided to the e2e test suite through the `WINDOWS_INSTANCES_DATA` environment
+variable.
+
+To deploy machines using the [infrastructure providers supported by WMCO](wmco-prerequisites.md#supported-cloud-providers-based-on-okdocp-version-and-wmco-version),
+refer to the specific provider documentation. See Windows instance [pre-requisites](https://github.com/openshift/windows-machine-config-operator/blob/master/docs/byoh-instance-pre-requisites.md).
+
+The information you need to collect from each Windows instance is:
+- the internal IP address
+- the Windows administrator username
+
+Export `WINDOWS_INSTANCES_DATA` as an environment variable with the corresponding [windows-instances ConfigMap](https://github.com/openshift/windows-machine-config-operator#adding-instances)
+data section, and a new `windows-instances` ConfigMap will be created during the execution of the e2e test suite.
+
+For example:
+```shell
+export WINDOWS_INSTANCES_DATA='
+data:
+  10.1.42.1: |-
+    username=Administrator
+'
+```
+where `10.1.42.1` is the IP address and `Administrator` is the Windows' administrator username of the Windows instance.
+
+After the `WINDOWS_INSTANCES_DATA` environment variable is set and exported you can run:
+```shell
+hack/run-ci-e2e-test.sh
+```
+
 ## Bundling the Windows Machine Config Operator
 This directory contains resources related to installing the WMCO onto a cluster using OLM.
 

docs/vsphere_ci/platform-none/README.md

@@ -0,0 +1,24 @@
+# Adding PTR records for platform=none CI clusters in vSphere
+
+WMCO's CSRs approver requires reverse DNS lookup for each Windows instance that 
+joins the cluster as a Windows worker. For CI clusters with platform-agnostic
+infrastructure (platform=none) a fixed lease pool is configured to ensure
+reverse DNS lookup is possible.
+
+The [create-ptr-records.sh](create-ptr-records.sh) script creates the reverse
+DNS lookup by creating PTR records for each IP address available in the selected
+subnets.
+
+Before running the [create-ptr-records.sh](create-ptr-records.sh) script, ensure
+AWS CLI is properly configured with AWS credentials and region for
+`openshift-vmware-cloud-ci` account, for example:
+```shell
+# configures AWS CLI
+export AWS_PROFILE="openshift-vmware-cloud-ci"
+export AWS_REGION="us-west-2"
+
+# run script
+./create-ptr-records.sh
+```
+where `openshift-vmware-cloud-ci` is the name of profile that contains the
+credentials, and `us-west-2` is the region where the resources were provisioned.

docs/vsphere_ci/platform-none/create-ptr-records.sh

@@ -0,0 +1,107 @@
+#!/bin/bash
+
+# create-ptr-records.sh - Create PTR records in AWS Route53 for vSphere CI.
+# For platform=none clusters the following ci-segments are reserved:
+#  - ci-segment-56
+#  - ci-segment-57
+#  - ci-segment-58
+#  - ci-segment-59
+# each segment sits on a `/27` subnet (`192.168.x.1/27`) with DHCP range from
+# `192.168.x.10` to `192.168.x.30`, where `x` accounts for the third octet and
+# matches the number of the ci-segment, from 56 to 59 inclusive. DNS is provided
+# by the VPC with server IP `10.0.0.2`.
+#
+# USAGE
+#    create-ptr-records.sh
+#
+# PREREQUISITES
+#    aws      to fetch VPC information and create hosted zone with record sets
+
+# name of the VPC cloud formation stack
+VSPHERE_VPC_STACK_NAME="vsphere-vpc"
+
+# ci-segments for platform=none in vSphere
+CI_SEGMENTS=(56 57 58 59)
+
+# start and end IP addresses of the subnet
+SUBNET_START=10
+SUBNET_END=30
+
+# createJSONRecordSets creates the JSON batch file with the PTR record sets
+# for the given IP range.
+function createJSONRecordSets () {
+    batch_file=$1; third_octet=$2
+    # init change_batch.json file content
+    cat <<EOF > ${batch_file}
+{
+  "Comment": "PTR records for ci-segment-${third_octet}",
+  "Changes": [
+EOF
+    # loop ip range
+    for ip in $(seq $SUBNET_START $SUBNET_END); do
+        # append record set
+        cat <<EOF >> ${batch_file}
+    {
+      "Action": "CREATE",
+      "ResourceRecordSet": {
+        "Name": "${ip}.${third_octet}.168.192.in-addr.arpa",
+        "Type": "PTR",
+        "TTL": 172800,
+        "ResourceRecords": [
+          {
+            "Value": "192.168.${third_octet}.${ip}"
+          }
+        ]
+      }
+    }$([[ $ip = $SUBNET_END ]] && echo "" || echo ",")
+EOF
+    done
+    cat <<EOF >> ${batch_file}
+  ]
+}
+EOF
+}
+
+# find the VPC ID
+VPC_ID=$(aws cloudformation describe-stacks \
+    --region $AWS_REGION \
+    --stack-name $VSPHERE_VPC_STACK_NAME \
+    | jq -r '.Stacks[0].Outputs[] | select(.OutputKey | contains("VpcId")).OutputValue') || {
+    echo "Error getting vSphere VPC ID"
+    exit 1
+}
+
+# create temp file for batch data
+batch_file=$(mktemp)
+
+# loop ci-segments
+for third_octet in "${CI_SEGMENTS[@]}"; do
+    echo "Processing ci-segment-$third_octet"
+    echo "Creating private hosted zone associate to VPC $AWS_REGION/$VPC_ID"
+    hosted_zone_id=$(aws route53 create-hosted-zone \
+        --name "${third_octet}".168.192.in-addr.arpa \
+        --hosted-zone-config PrivateZone=true,Comment="Hosted zone for VMC ci-segment-${third_octet}" \
+        --caller-reference "$(uuidgen)" \
+        --vpc VPCRegion="$AWS_REGION",VPCId="$VPC_ID" | jq -r '.HostedZone.Id') || {
+            echo "Error creating hosted zone for ci-segment-${third_octet}"
+            exit 1
+        }
+    echo "Created hosted zone $hosted_zone_id for ci-segment-${third_octet}"
+
+    echo "Creating PTR record sets batch file for ci-segment-${third_octet}"
+    createJSONRecordSets "${batch_file}" "${third_octet}"
+
+    echo "Adding PTR record sets to hosted zone ${hosted_zone_id}"
+    aws route53 change-resource-record-sets \
+      --hosted-zone-id "${hosted_zone_id}" \
+      --change-batch file://"${batch_file}" || {
+        echo "Error creating PTR record sets for hosted zone ${hosted_zone_id} from batch file ${batch_file}"
+        exit 1
+      }
+done
+
+# cleanup
+rm -f "${batch_file}"
+
+# success
+exit 0

hack/common.sh

@@ -167,3 +167,29 @@ transform_csv() {
   fi
   sed -i "s|"$1"|"$2"|g" $MANIFEST_LOC/manifests/windows-machine-config-operator.clusterserviceversion.yaml
 }
+
+# creates the `windows-instances` ConfigMap
+# Parameters:
+#  1: the ConfigMap data section
+createWindowsInstancesConfigMap() {
+  DATA=$1
+  if [[ -z "$DATA" ]]; then
+    error-exit "ConfigMap data cannot be empty"
+  fi
+  cat <<EOF | oc apply -f -
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: windows-instances
+  namespace: ${WMCO_DEPLOY_NAMESPACE}
+${DATA}
+EOF
+}
+
+# returns the number of instances from `windows-instances` ConfigMap
+getWindowsInstanceCountFromConfigMap() {
+ oc get configmaps \
+   windows-instances \
+   -n "${WMCO_DEPLOY_NAMESPACE}" \
+   -o json | jq '.data | length'
+}

hack/run-ci-e2e-test.sh

@@ -101,6 +101,25 @@ if ! [[ "$OPENSHIFT_CI" = "true" &&  "$TEST" = "upgrade" ]]; then
   done
 fi
 
+# WINDOWS_INSTANCES_DATA holds the windows-instances ConfigMap data section
+WINDOWS_INSTANCES_DATA=${WINDOWS_INSTANCES_DATA:-}
+# Check WINDOWS_INSTANCES_DATA and create the windows-instances ConfigMap, if
+# any. The ConfigMap creation must takes place after the operator's deployment
+# process is complete, because the namespace is removed as part of the cleanup
+# step (cleanup_WMCO) while retrying the deployment.
+if [[ -n "$WINDOWS_INSTANCES_DATA" ]]; then
+  createWindowsInstancesConfigMap "${WINDOWS_INSTANCES_DATA}" || {
+    error-exit "error creating windows-instances ConfigMap with ${WINDOWS_INSTANCES_DATA}"
+  }
+  # read BYOH from ConfigMap
+  BYOH_NODE_COUNT=$(getWindowsInstanceCountFromConfigMap) || {
+    error-exit "error getting windows-instances ConfigMap"
+  }
+  # update BYOH node count option or default to 0
+  BYOH_NODE_COUNT_OPTION="--byoh-node-count=${BYOH_NODE_COUNT:-0}"
+  echo "updated ${BYOH_NODE_COUNT_OPTION}"
+fi
+
 # The bool flags in golang does not respect key value pattern. They follow -flag=x pattern.
 # -flag x is allowed for non-boolean flags only(https://golang.org/pkg/flag/)
 

test/e2e/create_test.go

@@ -108,6 +108,23 @@ func (tc *testContext) createWindowsInstanceConfigMap(machines *mapi.MachineList
 	return nil
 }
 
+// validateWindowsInstanceConfigMap validates the windows-instance ConfigMap
+func (tc *testContext) validateWindowsInstanceConfigMap(expectedCount int) error {
+	windowsInstances, err := tc.client.K8s.CoreV1().ConfigMaps(tc.namespace).Get(context.TODO(),
+		wiparser.InstanceConfigMap, metav1.GetOptions{})
+	if err != nil {
+		return errors.Wrapf(err, "error retrieving ConfigMap: %s", wiparser.InstanceConfigMap)
+	}
+	// check instance count
+	actualCount := len(windowsInstances.Data)
+	if actualCount != expectedCount {
+		return errors.Wrapf(err, "invalid BYOH instance count: %v, expected: %v", actualCount, expectedCount)
+	}
+	// TODO: Validate windowsInstances.Data (See https://issues.redhat.com/browse/WINC-671)
+	// ConfigMap is valid, return no error
+	return nil
+}
+
 // testMachineConfiguration tests that the Windows Machine controller can properly configure Machines
 func (tc *testContext) testMachineConfiguration(t *testing.T) {
 	if gc.numberOfMachineNodes == 0 {
@@ -139,14 +156,22 @@ func (tc *testContext) testBYOHConfiguration(t *testing.T) {
 		t.Skip("BYOH testing disabled")
 	}
 
-	err := tc.disableClusterMachineApprover()
-	require.NoError(t, err, "failed to scale down Machine Approver pods")
-
-	err = tc.provisionBYOHConfigMapWithMachineSet()
-	require.NoError(t, err, "error provisioning BYOH ConfigMap with MachineSets")
-
+	// For platform-agnostic infrastructure just validate the BYOH ConfigMap
+	// TODO: See https://github.com/openshift/windows-machine-config-operator/pull/858#discussion_r780316359
+	if tc.CloudProvider.GetType() == config.NonePlatformType {
+		err := tc.validateWindowsInstanceConfigMap(int(gc.numberOfBYOHNodes))
+		require.NoError(t, err, "error validating windows-instances ConfigMap")
+		log.Printf("using %v BYOH instance(s) already provisioned", gc.numberOfBYOHNodes)
+	} else {
+		// Otherwise, provision BYOH instances with MachineSet
+		err := tc.disableClusterMachineApprover()
+		require.NoError(t, err, "failed to scale down Machine Approver pods")
+		err = tc.provisionBYOHConfigMapWithMachineSet()
+		require.NoError(t, err, "error provisioning BYOH ConfigMap with MachineSets")
+	}
+	// Wait for Windows worker node to become available
 	t.Run("VM is configured by ConfigMap controller", func(t *testing.T) {
-		err = tc.waitForWindowsNodes(gc.numberOfBYOHNodes, false, false, true)
+		err := tc.waitForWindowsNodes(gc.numberOfBYOHNodes, false, false, true)
 		assert.NoError(t, err, "Windows node creation failed")
 	})
 }

test/e2e/providers/cloudprovider.go

@@ -10,6 +10,7 @@ import (
 	oc "github.com/openshift/windows-machine-config-operator/test/e2e/clusterinfo"
 	awsProvider "github.com/openshift/windows-machine-config-operator/test/e2e/providers/aws"
 	azureProvider "github.com/openshift/windows-machine-config-operator/test/e2e/providers/azure"
+	noneProvider "github.com/openshift/windows-machine-config-operator/test/e2e/providers/none"
 	vSphereProvider "github.com/openshift/windows-machine-config-operator/test/e2e/providers/vsphere"
 )
 
@@ -37,6 +38,8 @@ func NewCloudProvider(hasCustomVXLANPort bool) (CloudProvider, error) {
 		return azureProvider.New(openshift, hasCustomVXLANPort)
 	case config.VSpherePlatformType:
 		return vSphereProvider.New(openshift)
+	case config.NonePlatformType:
+		return noneProvider.New(openshift)
 	default:
 		return nil, fmt.Errorf("the '%v' cloud provider is not supported", provider)
 	}

test/e2e/providers/none/none.go

@@ -0,0 +1,31 @@
+package none
+
+import (
+	"github.com/pkg/errors"
+
+	config "github.com/openshift/api/config/v1"
+	mapi "github.com/openshift/machine-api-operator/pkg/apis/machine/v1beta1"
+	"github.com/openshift/windows-machine-config-operator/test/e2e/clusterinfo"
+)
+
+// Provider is a provider struct for testing platform=none
+type Provider struct {
+	oc *clusterinfo.OpenShift
+}
+
+// New returns a Provider implementation for platform=none
+func New(clientset *clusterinfo.OpenShift) (*Provider, error) {
+	return &Provider{
+		oc: clientset,
+	}, nil
+}
+
+// GenerateMachineSet is not supported for platform=none and throws an exception
+func (p *Provider) GenerateMachineSet(withWindowsLabel bool, replicas int32) (*mapi.MachineSet, error) {
+	return nil, errors.New("MachineSet generation not supported for platform=none")
+}
+
+// GetType returns the platform type for platform=none
+func (p *Provider) GetType() config.PlatformType {
+	return config.NonePlatformType
+}