Merge pull request #2024 from openshift-cherrypick-robot/cherry-pick-…

…2006-to-release-4.15

[release-4.15] OCPBUGS-27503: Use DSR load balancing in kube-proxy

cmd/operator/main.go

@@ -132,7 +132,7 @@ func main() {
 	}
 
 	if err := payload.PopulateNetworkConfScript(clusterConfig.Network().GetServiceCIDR(), windows.OVNKubeOverlayNetwork,
-		windows.HNSPSModule, windows.CniConfDir+"\\cni.conf"); err != nil {
+		windows.HNSPSModule, windows.CNIDir, windows.CniConfDir+"\\cni.conf"); err != nil {
 		setupLog.Error(err, "unable to generate CNI config script")
 		os.Exit(1)
 	}

docs/byoh-instance-pre-requisites.md

@@ -2,8 +2,9 @@
 
 The following pre-requisites must be fulfilled in order to add a Windows BYOH node.
 * The instance must be on the same network as the Linux worker nodes in the cluster.
-* Port 22 must be open and running [an SSH server](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse).
-* Port 10250 must be open in order for log collection to function.
+* Port 22 must allow inbound TCP traffic and be running [an SSH server](https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse).
+* Port 9182 must allow inbound TCP traffic in order for node and pod metrics collection to function.
+* Port 10250 must allow inbound TCP traffic in order for log collection to function.
 * An administrator user is present with the [private key used in the secret](/README.md#create-a-private-key-secret) set as an authorized SSH key.
 * The hostname of the instance must follow the [RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123) DNS label standard:
   * Contain only lowercase alphanumeric characters or '-'.

docs/vsphere-golden-image.md

@@ -93,13 +93,13 @@ In case no firewall rule exist, you must create it by running the following Powe
     New-NetFirewallRule -DisplayName 'OpenSSH Server (sshd)' -LocalPort 22 -Enabled True -Direction Inbound -Protocol TCP -Action Allow 
 ```
 
-## 4. Set up incoming connection for container logs
+## 4. Set up incoming connection for container logs and metrics
 
-Create a new firewall rule in the Windows VM to allow incoming connections for container logs, usually 
-on TCP port `10250` by running the following PowerShell command:
+Create new firewall rules in the Windows VM to allow incoming connections for container logs and metrics:
 
 ```powershell
     New-NetFirewallRule -DisplayName "ContainerLogsPort" -LocalPort 10250 -Enabled True -Direction Inbound -Protocol TCP -Action Allow -EdgeTraversalPolicy Allow
+    New-NetFirewallRule -DisplayName "WindowsExporter" -LocalPort 9182 -Enabled True -Direction Inbound -Protocol TCP -Action Allow -EdgeTraversalPolicy Allow
 ```
 
 ## 5. Install Windows OS updates

docs/vsphere_ci/scripts/install-firewall-rules.ps1

@@ -3,8 +3,9 @@
 # USAGE
 #    ./install-firewall-rules.ps1 
 
-# create firewall rule to allow Container Logs on port 10250
+# Allow incoming connections for container logs and metrics
 New-NetFirewallRule -DisplayName "ContainerLogsPort" -LocalPort 10250 -Enabled True -Direction Inbound -Protocol TCP -Action Allow -EdgeTraversalPolicy Allow
+New-NetFirewallRule -DisplayName "WindowsExporter" -LocalPort 9182 -Enabled True -Direction Inbound -Protocol TCP -Action Allow -EdgeTraversalPolicy Allow
 
 # success
 exit 0

pkg/nodeconfig/payload/payload.go

@@ -130,6 +130,48 @@ $cni_template=@'
 }
 '@
 
+# from https://github.com/kubernetes-sigs/sig-windows-tools/blob/fbe00b42e2a5cca06bc182e1b6ee579bd65ed1b5/hostprocess/flannel/kube-proxy/start.ps1
+function GetSourceVip($NetworkName)
+{
+    mkdir -force CNI_DIR\sourcevip | Out-Null
+    $sourceVipJson = [io.Path]::Combine("CNI_DIR", "sourcevip",  "sourceVip.json")
+    $sourceVipRequest = [io.Path]::Combine("CNI_DIR", "sourcevip", "sourceVipRequest.json")
+
+    if (Test-Path $sourceVipJson) {
+        $sourceVipJSONData = Get-Content $sourceVipJson | ConvertFrom-Json
+        $vip = $sourceVipJSONData.ip4.ip.Split("/")[0]
+        return $vip
+    }
+
+    $hnsNetwork = Get-HnsNetwork | ? Name -EQ $NetworkName.ToLower()
+    $subnet = $hnsNetwork.Subnets[0].AddressPrefix
+
+    $ipamConfig = @"
+    {"cniVersion": "0.2.0", "name": "$NetworkName", "ipam":{"type":"host-local","ranges":[[{"subnet":"$subnet"}]],"dataDir":"/var/lib/cni/networks"}}
+"@
+
+    $ipamConfig | Out-File $sourceVipRequest
+
+    $env:CNI_COMMAND="ADD"
+    $env:CNI_CONTAINERID="dummy"
+    $env:CNI_NETNS="dummy"
+    $env:CNI_IFNAME="dummy"
+    $env:CNI_PATH="CNI_DIR"
+
+    # reserve an ip address for source VIP, a requirement for kubeproxy in overlay mode
+    Get-Content $sourceVipRequest | CNI_DIR\host-local.exe | Out-File $sourceVipJson
+
+    Remove-Item env:CNI_COMMAND
+    Remove-Item env:CNI_CONTAINERID
+    Remove-Item env:CNI_NETNS
+    Remove-Item env:CNI_IFNAME
+    Remove-Item env:CNI_PATH
+
+    $sourceVipJSONData = Get-Content $sourceVipJson | ConvertFrom-Json
+    $vip = $sourceVipJSONData.ip4.ip.Split("/")[0]
+    return $vip
+}
+
 # Generate CNI Config
 $hns_network=Get-HnsNetwork  | where { $_.Name -eq 'HNS_NETWORK'}
 $subnet=$hns_network.Subnets.AddressPrefix
@@ -146,15 +188,8 @@ if($existing_config -ne $cni_template){
     Set-Content -Path "CNI_CONFIG_PATH" -Value $cni_template -NoNewline
 }
 
-# Create HNS endpoint if it doesn't exist
-$endpoint = Invoke-HNSRequest GET endpoints | where { $_.Name -eq 'VIPEndpoint'}
-if( $endpoint -eq $null) {
-    $endpoint = New-HnsEndpoint -NetworkId $hns_network.ID -Name "VIPEndpoint"
-    Attach-HNSHostEndpoint -EndpointID $endpoint.ID -CompartmentID 1
-}
-
-# Return HNS endpoint IP
-(Get-NetIPConfiguration -AllCompartments -All -Detailed | where { $_.NetAdapter.LinkLayerAddress -eq $endpoint.MacAddress }).IPV4Address.IPAddress.Trim()
+# Return source VIP for HNS network
+(GetSourceVip("HNS_NETWORK"))
 `
 )
 
@@ -177,9 +212,9 @@ func NewFileInfo(path string) (*FileInfo, error) {
 }
 
 // PopulateNetworkConfScript creates the .ps1 file responsible for CNI configuration
-func PopulateNetworkConfScript(clusterCIDR, hnsNetworkName, hnsPSModulePath, cniConfigPath string) error {
+func PopulateNetworkConfScript(clusterCIDR, hnsNetworkName, hnsPSModulePath, cniDir, cniConfigPath string) error {
 	scriptContents, err := generateNetworkConfigScript(clusterCIDR, hnsNetworkName,
-		hnsPSModulePath, cniConfigPath)
+		hnsPSModulePath, cniDir, cniConfigPath)
 	if err != nil {
 		return err
 	}
@@ -188,13 +223,14 @@ func PopulateNetworkConfScript(clusterCIDR, hnsNetworkName, hnsPSModulePath, cni
 
 // generateNetworkConfigScript generates the contents of the .ps1 file responsible for CNI configuration
 func generateNetworkConfigScript(clusterCIDR, hnsNetworkName, hnsPSModulePath,
-	cniConfigPath string) (string, error) {
+	cniDir, cniConfig string) (string, error) {
 	networkConfScript := networkConfTemplate
 	for key, val := range map[string]string{
 		"HNS_NETWORK":          hnsNetworkName,
 		"SERVICE_NETWORK_CIDR": clusterCIDR,
 		"HNS_MODULE_PATH":      hnsPSModulePath,
-		"CNI_CONFIG_PATH":      cniConfigPath,
+		"CNI_DIR":              cniDir,
+		"CNI_CONFIG_PATH":      cniConfig,
 	} {
 		networkConfScript = strings.ReplaceAll(networkConfScript, key, val)
 	}

pkg/nodeconfig/payload/payload_test.go

@@ -64,6 +64,48 @@ $cni_template=@'
 }
 '@
 
+# from https://github.com/kubernetes-sigs/sig-windows-tools/blob/fbe00b42e2a5cca06bc182e1b6ee579bd65ed1b5/hostprocess/flannel/kube-proxy/start.ps1
+function GetSourceVip($NetworkName)
+{
+    mkdir -force c:\k\cni\sourcevip | Out-Null
+    $sourceVipJson = [io.Path]::Combine("c:\k\cni", "sourcevip",  "sourceVip.json")
+    $sourceVipRequest = [io.Path]::Combine("c:\k\cni", "sourcevip", "sourceVipRequest.json")
+
+    if (Test-Path $sourceVipJson) {
+        $sourceVipJSONData = Get-Content $sourceVipJson | ConvertFrom-Json
+        $vip = $sourceVipJSONData.ip4.ip.Split("/")[0]
+        return $vip
+    }
+
+    $hnsNetwork = Get-HnsNetwork | ? Name -EQ $NetworkName.ToLower()
+    $subnet = $hnsNetwork.Subnets[0].AddressPrefix
+
+    $ipamConfig = @"
+    {"cniVersion": "0.2.0", "name": "$NetworkName", "ipam":{"type":"host-local","ranges":[[{"subnet":"$subnet"}]],"dataDir":"/var/lib/cni/networks"}}
+"@
+
+    $ipamConfig | Out-File $sourceVipRequest
+
+    $env:CNI_COMMAND="ADD"
+    $env:CNI_CONTAINERID="dummy"
+    $env:CNI_NETNS="dummy"
+    $env:CNI_IFNAME="dummy"
+    $env:CNI_PATH="c:\k\cni"
+
+    # reserve an ip address for source VIP, a requirement for kubeproxy in overlay mode
+    Get-Content $sourceVipRequest | c:\k\cni\host-local.exe | Out-File $sourceVipJson
+
+    Remove-Item env:CNI_COMMAND
+    Remove-Item env:CNI_CONTAINERID
+    Remove-Item env:CNI_NETNS
+    Remove-Item env:CNI_IFNAME
+    Remove-Item env:CNI_PATH
+
+    $sourceVipJSONData = Get-Content $sourceVipJson | ConvertFrom-Json
+    $vip = $sourceVipJSONData.ip4.ip.Split("/")[0]
+    return $vip
+}
+
 # Generate CNI Config
 $hns_network=Get-HnsNetwork  | where { $_.Name -eq 'OVNKubernetesHNSNetwork'}
 $subnet=$hns_network.Subnets.AddressPrefix
@@ -73,25 +115,18 @@ $cni_template=$cni_template.Replace("provider_address",$provider_address)
 
 # Compare CNI config with existing file, and replace if necessary
 $existing_config=""
-if(Test-Path -Path c:\k\cni.conf) {
-` + "    $existing_config=((Get-Content -Path \"c:\\k\\cni.conf\" -Raw) -Replace \"`r\",\"\")" + `
+if(Test-Path -Path c:\k\cni\config\cni.conf) {
+` + "    $existing_config=((Get-Content -Path \"c:\\k\\cni\\config\\cni.conf\" -Raw) -Replace \"`r\",\"\")" + `
 }
 if($existing_config -ne $cni_template){
-    Set-Content -Path "c:\k\cni.conf" -Value $cni_template -NoNewline
-}
-
-# Create HNS endpoint if it doesn't exist
-$endpoint = Invoke-HNSRequest GET endpoints | where { $_.Name -eq 'VIPEndpoint'}
-if( $endpoint -eq $null) {
-    $endpoint = New-HnsEndpoint -NetworkId $hns_network.ID -Name "VIPEndpoint"
-    Attach-HNSHostEndpoint -EndpointID $endpoint.ID -CompartmentID 1
+    Set-Content -Path "c:\k\cni\config\cni.conf" -Value $cni_template -NoNewline
 }
 
-# Return HNS endpoint IP
-(Get-NetIPConfiguration -AllCompartments -All -Detailed | where { $_.NetAdapter.LinkLayerAddress -eq $endpoint.MacAddress }).IPV4Address.IPAddress.Trim()
+# Return source VIP for HNS network
+(GetSourceVip("OVNKubernetesHNSNetwork"))
 `
 	actual, err := generateNetworkConfigScript("10.0.0.1/32",
-		"OVNKubernetesHNSNetwork", "c:\\k\\hns.psm1", "c:\\k\\cni.conf")
+		"OVNKubernetesHNSNetwork", "c:\\k\\hns.psm1", "c:\\k\\cni", "c:\\k\\cni\\config\\cni.conf")
 	require.NoError(t, err)
 	assert.Equal(t, string(expectedOut), actual)
 }

pkg/secrets/secrets.go

@@ -61,6 +61,7 @@ func GenerateUserData(platformType oconfig.PlatformType, publicKey ssh.PublicKey
 
 // generateUserDataWithPubKey returns the Windows user data for the given pubKey
 func generateUserDataWithPubKey(pubKey string) string {
+	windowsExporterPort := "9182"
 	return `function Get-RandomPassword {
 				Add-Type -AssemblyName 'System.Web'
 				return [System.Web.Security.Membership]::GeneratePassword(16, 2)
@@ -82,6 +83,7 @@ func generateUserDataWithPubKey(pubKey string) string {
 			$firewallRuleName = "ContainerLogsPort"
 			$containerLogsPort = "10250"
 			New-NetFirewallRule -DisplayName $firewallRuleName -Direction Inbound -Action Allow -Protocol TCP -LocalPort $containerLogsPort -EdgeTraversalPolicy Allow
+			New-NetFirewallRule -DisplayName "WindowsExporter" -Direction Inbound -Action Allow -Protocol TCP -LocalPort "` + windowsExporterPort + `" -EdgeTraversalPolicy Allow
 			Set-Service -Name sshd -StartupType 'Automatic'
 			Start-Service sshd
 			(Get-Content -path C:\ProgramData\ssh\sshd_config) | ForEach-Object {

pkg/services/services.go

@@ -134,9 +134,9 @@ func hybridOverlayConfiguration(vxlanPort string, debug bool) servicescm.Service
 // kubeProxyConfiguration returns the Service definition for kube-proxy
 func kubeProxyConfiguration(debug bool) servicescm.Service {
 	sanitizedSubnetAnnotation := strings.ReplaceAll(nodeconfig.HybridOverlaySubnet, ".", "\\.")
-	cmd := fmt.Sprintf("%s -log-file=%s %s --windows-service --proxy-mode=kernelspace --feature-gates=WinOverlay=true "+
+	cmd := fmt.Sprintf("%s -log-file=%s %s --windows-service --proxy-mode=kernelspace --feature-gates=WinOverlay=true,WinDSR=true "+
 		"--hostname-override=NODE_NAME --kubeconfig=%s --cluster-cidr=NODE_SUBNET "+
-		"--network-name=%s --source-vip=ENDPOINT_IP --enable-dsr=false", windows.KubeLogRunnerPath, windows.KubeProxyLog,
+		"--network-name=%s --source-vip=ENDPOINT_IP --enable-dsr=true", windows.KubeLogRunnerPath, windows.KubeProxyLog,
 		windows.KubeProxyPath, windows.KubeconfigPath, windows.OVNKubeOverlayNetwork)
 	// Set log level
 	cmd = fmt.Sprintf("%s %s", cmd, klogVerbosityArg(debug))

pkg/windows/windows.go

@@ -46,10 +46,10 @@ const (
 	HybridOverlayLogDir = logDir + "\\hybrid-overlay"
 	// wicdLogDir is the remote wicd log directory
 	wicdLogDir = logDir + "\\wicd"
-	// cniDir is the directory for storing CNI binaries
-	cniDir = K8sDir + "\\cni"
+	// CNIDir is the directory for storing CNI binaries
+	CNIDir = K8sDir + "\\cni"
 	// CniConfDir is the directory for storing CNI configuration
-	CniConfDir = cniDir + "\\config"
+	CniConfDir = CNIDir + "\\config"
 	// ContainerdDir is the directory for storing Containerd binary
 	ContainerdDir = K8sDir + "\\containerd"
 	// ContainerdPath is the location of the containerd exe
@@ -151,7 +151,7 @@ var (
 	// RequiredDirectories is a list of directories to be created by WMCO
 	RequiredDirectories = []string{
 		remoteDir,
-		cniDir,
+		CNIDir,
 		CniConfDir,
 		logDir,
 		KubeletLogDir,
@@ -188,9 +188,9 @@ func getFilesToTransfer(platform *config.PlatformType) map[string]string {
 		payload.HybridOverlayPath:              K8sDir,
 		payload.HNSPSModule:                    remoteDir,
 		payload.WindowsExporterPath:            K8sDir,
-		payload.WinBridgeCNIPlugin:             cniDir,
-		payload.HostLocalCNIPlugin:             cniDir,
-		payload.WinOverlayCNIPlugin:            cniDir,
+		payload.WinBridgeCNIPlugin:             CNIDir,
+		payload.HostLocalCNIPlugin:             CNIDir,
+		payload.WinOverlayCNIPlugin:            CNIDir,
 		payload.KubeProxyPath:                  K8sDir,
 		payload.KubeletPath:                    K8sDir,
 		payload.KubeLogRunnerPath:              K8sDir,

test/e2e/create_test.go

@@ -33,10 +33,6 @@ const (
 	// vmConfigurationTime is the maximum amount of time expected for a Windows VM to be fully configured and ready for WMCO
 	// after the hardware is provisioned.
 	vmConfigurationTime = 10 * time.Minute
-
-	machineApproverNamespace   = "openshift-cluster-machine-approver"
-	machineApproverDeployment  = "machine-approver"
-	machineApproverPodSelector = "app=machine-approver"
 )
 
 func creationTestSuite(t *testing.T) {
@@ -348,9 +344,7 @@ func (tc *testContext) disableClusterMachineApprover() error {
 	// Scale the Cluster Machine Approver Deployment to 0
 	// This is required for testing BYOH CSR approval feature so that BYOH instances
 	// CSR's are not approved by Cluster Machine Approver
-	expectedPodCount := int32(0)
-	return tc.scaleDeployment(machineApproverNamespace, machineApproverDeployment, machineApproverPodSelector,
-		&expectedPodCount)
+	return tc.scaleMachineApprover(0)
 }
 
 // setPowerShellDefaultShell changes the instance backed by the given Machine to have a default SSH shell of PowerShell
@@ -601,3 +595,10 @@ func (tc *testContext) scaleDeployment(namespace, name, selector string, expecte
 	}
 	return nil
 }
+
+// scaleMachineApprover scales the machine-approver deployment to the given replica count
+func (tc *testContext) scaleMachineApprover(replicas int) error {
+	replicaCount := int32(replicas)
+	return tc.scaleDeployment("openshift-cluster-machine-approver", "machine-approver", "app=machine-approver",
+		&replicaCount)
+}

test/e2e/network_test.go

@@ -501,7 +501,7 @@ func (tc *testContext) createWindowsServerDeployment(name string, command []stri
 	deploymentsClient := tc.client.K8s.AppsV1().Deployments(tc.workloadNamespace)
 	replicaCount := int32(1)
 	// affinity being nil is a hint that the caller does not care which nodes the pods are deployed to
-	if affinity == nil {
+	if affinity == nil && volumes == nil {
 		replicaCount = int32(3)
 	}
 	windowsServerImage := tc.getWindowsServerContainerImage()

test/e2e/storage_test.go

@@ -55,12 +55,10 @@ func testStorage(t *testing.T) {
 		}()
 	}
 	pvcVolumeSource := &core.PersistentVolumeClaimVolumeSource{ClaimName: pvc.GetName()}
-	affinity, err := getAffinityForNode(&gc.allNodes()[0])
-	require.NoError(t, err)
 
 	// The deployment will not come to ready if the volume is not able to be attached to the pod. If the deployment is
 	// successful, storage is working as expected.
-	winServerDeployment, err := tc.deployWindowsWebServer("win-webserver-storage-test", affinity, pvcVolumeSource)
+	winServerDeployment, err := tc.deployWindowsWebServer("win-webserver-storage-test", nil, pvcVolumeSource)
 	require.NoError(t, err)
 	if err == nil && !skipWorkloadDeletion {
 		defer func() {

test/e2e/upgrade_test.go

@@ -208,6 +208,12 @@ func (tc *testContext) deployWindowsWorkloadAndTester() (func(), error) {
 func TestUpgrade(t *testing.T) {
 	tc, err := NewTestContext()
 	require.NoError(t, err)
+
+	// In the case that upgrading a Machine node require the deletion of the VM, bootstrap CSRs will need to be approved
+	// Ensure the machine approver is scaled, as it may not be depending on the order of tests ran
+	err = tc.scaleMachineApprover(1)
+	require.NoError(t, err)
+
 	err = tc.waitForConfiguredWindowsNodes(int32(numberOfMachineNodes), true, false)
 	assert.NoError(t, err, "timed out waiting for Windows Machine nodes")
 	err = tc.waitForConfiguredWindowsNodes(int32(numberOfBYOHNodes), true, true)

test/e2e/validation_test.go

@@ -765,10 +765,8 @@ func (tc *testContext) testCSRApproval(t *testing.T) {
 		}
 	}
 
-	// Scale the Cluster Machine Approver deployment back to 1.
-	expectedPodCount := int32(1)
-	err := tc.scaleDeployment(machineApproverNamespace, machineApproverDeployment, machineApproverPodSelector,
-		&expectedPodCount)
+	// Revert changes to the cluster machine approver
+	err := tc.scaleMachineApprover(1)
 	require.NoError(t, err, "failed to scale up Cluster Machine Approver pods")
 }