From 5895c7815a49ccda147b39bed7b18bc7c677b655 Mon Sep 17 00:00:00 2001 From: Sebastian Soto Date: Thu, 4 Aug 2022 11:00:59 -0400 Subject: [PATCH] kube proxy --- build/Dockerfile | 4 +- build/Dockerfile.ci | 5 ++ build/Dockerfile.wmco | 7 ++ ...config-operator.clusterserviceversion.yaml | 2 +- cmd/operator/main.go | 24 +++++- pkg/internal/cni-conf-template.ps1 | 81 +++++++++++++++++++ pkg/nodeconfig/nodeconfig.go | 5 +- pkg/nodeconfig/payload/payload.go | 9 ++- pkg/services/services.go | 35 ++++++++ pkg/windows/windows.go | 50 ++++++------ 10 files changed, 185 insertions(+), 37 deletions(-) create mode 100644 pkg/internal/cni-conf-template.ps1 diff --git a/build/Dockerfile b/build/Dockerfile index c69e3e9311..d03a909b8d 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -88,7 +88,6 @@ RUN make build-daemon #│ ├── host-local.exe #│ ├── win-bridge.exe #│ ├── win-overlay.exe -#│ └── cni-conf-template.json #├── containerd #│ ├── containerd.exe #│ └── containerd-shim-runhcs-v1.exe @@ -99,6 +98,7 @@ RUN make build-daemon #│ └── kube-proxy.exe #├── powershell #│ └── wget-ignore-cert.ps1 +#│ └── cni-conf-template.ps1 #│ └── hns.psm1 #├── windows_exporter.exe #├── windows-instance-config-daemon.exe @@ -139,7 +139,7 @@ WORKDIR /payload/cni/ COPY --from=build /build/windows-machine-config-operator/containernetworking-plugins/bin/host-local.exe . COPY --from=build /build/windows-machine-config-operator/containernetworking-plugins/bin/win-bridge.exe . COPY --from=build /build/windows-machine-config-operator/containernetworking-plugins/bin/win-overlay.exe . -COPY pkg/internal/cni-conf-template.json . +COPY pkg/internal/cni-conf-template.ps1 . # Copy required powershell scripts WORKDIR /payload/powershell/ diff --git a/build/Dockerfile.ci b/build/Dockerfile.ci index b76fb42ae4..fa901479a7 100644 --- a/build/Dockerfile.ci +++ b/build/Dockerfile.ci @@ -158,9 +158,14 @@ COPY --from=build /build/windows-machine-config-operator/containernetworking-plu COPY --from=build /build/windows-machine-config-operator/containernetworking-plugins/bin/win-overlay.exe . COPY --from=build /build/windows-machine-config-operator/pkg/internal/cni-conf-template.json . +# Created directory for generated files with open permissions +RUN mkdir /payload/generated +RUN chmod 0777 /payload/generated + # Copy required powershell scripts WORKDIR /payload/powershell/ COPY --from=build /build/windows-machine-config-operator/pkg/internal/wget-ignore-cert.ps1 . +COPY pkg/internal/cni-conf-template.ps1 . COPY --from=build /build/windows-machine-config-operator/pkg/internal/hns.psm1 . WORKDIR / diff --git a/build/Dockerfile.wmco b/build/Dockerfile.wmco index 9348f12225..5708ceda54 100644 --- a/build/Dockerfile.wmco +++ b/build/Dockerfile.wmco @@ -29,11 +29,18 @@ LABEL stage=operator WORKDIR /payload/ COPY --from=build /build/windows-machine-config-operator/build/_output/bin/windows-instance-config-daemon.exe . +# Created directory for generated files with open permissions +RUN mkdir generated +RUN chmod 0777 generated + + # Copy required powershell scripts WORKDIR /payload/powershell/ COPY pkg/internal/wget-ignore-cert.ps1 . +COPY pkg/internal/cni-conf-template.ps1 . COPY pkg/internal/hns.psm1 . + WORKDIR / ENV OPERATOR=/usr/local/bin/windows-machine-config-operator \ diff --git a/bundle/manifests/windows-machine-config-operator.clusterserviceversion.yaml b/bundle/manifests/windows-machine-config-operator.clusterserviceversion.yaml index 9a331b809c..8fd749248a 100644 --- a/bundle/manifests/windows-machine-config-operator.clusterserviceversion.yaml +++ b/bundle/manifests/windows-machine-config-operator.clusterserviceversion.yaml @@ -365,7 +365,7 @@ spec: fieldPath: metadata.name - name: OPERATOR_NAME value: windows-machine-config-operator - image: REPLACE_IMAGE + image: quay.io/ssoto/test:wmco imagePullPolicy: IfNotPresent name: manager resources: {} diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 7cfe1f1588..2a8f294473 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -4,6 +4,9 @@ import ( "context" "flag" "fmt" + "github.com/openshift/windows-machine-config-operator/pkg/windows" + "io/fs" + "io/ioutil" "os" "strings" @@ -114,13 +117,17 @@ func main() { payload.IgnoreWgetPowerShellPath, payload.WmcbPath, payload.WICDPath, - payload.CNIConfigTemplatePath, + payload.CNIConfigScriptTemplatePath, payload.HNSPSModule, payload.WindowsExporterPath, payload.AzureCloudNodeManagerPath, } if err := checkIfRequiredFilesExist(requiredFiles); err != nil { - setupLog.Error(err, "could not start the operator") + setupLog.Error(err, "payload missing files") + os.Exit(1) + } + if err := generateCNIConfScript(clusterConfig.Network().GetServiceCIDR()); err != nil { + setupLog.Error(err, "unable to generate CNI config script") os.Exit(1) } @@ -258,6 +265,19 @@ func checkIfRequiredFilesExist(requiredFiles []string) error { return nil } +// generateCNIConfScripts generates the .ps1 file responsible for CNI configuration generation +func generateCNIConfScript(clusterCIDR string) error { + out, err := ioutil.ReadFile(payload.CNIConfigScriptTemplatePath) + if err != nil { + return err + } + cniConfScript := strings.ReplaceAll(string(out), "HNS_NETWORK", windows.OVNKubeOverlayNetwork) + cniConfScript = strings.ReplaceAll(cniConfScript, "SERVICE_NETWORK_CIDR", clusterCIDR) + cniConfScript = strings.ReplaceAll(cniConfScript, "HNS_MODULE_PATH", windows.HNSPSModule) + cniConfScript = strings.ReplaceAll(cniConfScript, "CNI_CONFIG_PATH", windows.CniConfDir+"\\cni.conf") + return ioutil.WriteFile(payload.CNIConfigScriptPath, []byte(cniConfScript), fs.ModePerm) +} + // getWatchNamespace returns the Namespace the operator should be watching for changes // An empty value means the operator is running with cluster scope. func getWatchNamespace() (string, error) { diff --git a/pkg/internal/cni-conf-template.ps1 b/pkg/internal/cni-conf-template.ps1 new file mode 100644 index 0000000000..c502011947 --- /dev/null +++ b/pkg/internal/cni-conf-template.ps1 @@ -0,0 +1,81 @@ +# This script ensures the contents of the CNI config file is correct, and returns the HNS endpoint IP + +Import-Module -DisableNameChecking HNS_MODULE_PATH + +$cni_template=@' +{ + "CniVersion":"0.2.0", + "Name":"HNS_NETWORK", + "Type":"win-overlay", + "apiVersion": 2, + "Capabilities":{ + "portMappings": true, + "Dns":true + }, + "Ipam":{ + "Type":"host-local", + "Subnet":"ovn_host_subnet" + }, + "Policies":[ + { + "Name": "EndpointPolicy", + "Value": { + "Type": "OutBoundNAT", + "Settings": { + "ExceptionList": [ + "SERVICE_NETWORK_CIDR" + ], + "DestinationPrefix": "", + "NeedEncap": false + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "SDNROUTE", + "Settings": { + "ExceptionList": [], + "DestinationPrefix": "SERVICE_NETWORK_CIDR", + "NeedEncap": true + } + } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ProviderAddress", + "Settings": { + "ProviderAddress": "provider_address" + } + } + } + ] +} +'@ + +# Generate CNI Config +$hns_network=Get-HnsNetwork | where { $_.Name -eq 'HNS_NETWORK'} +$subnet=$hns_network.Subnets.AddressPrefix +$cni_template -replace "ovn_host_subnet",$subnet | Out-Null +$provider_address=$hns_network.ManagementIP +$cni_template -replace "provider_address",$provider_address | Out-Null + +# Compare CNI config with existing file, and replace if necessary +$existing_config="" +if(Test-Path -Path CNI_CONFIG_PATH) { + $existing_config= Get-Content -Path "CNI_CONFIG_PATH" +} +if($existing_config -ne $cni_template){ + Set-Content -Path "CNI_CONFIG_PATH" -Value $cni_template -NoNewline +} + +# Create HNS endpoint if it doesn't exist +$endpoint = Invoke-HNSRequest GET endpoints | where { $_.Name -eq 'VIPEndpoint'} +if( $endpoint -eq $null) { + $endpoint = New-HnsEndpoint -NetworkId $hns_network.ID -Name "VIPEndpoint" + Attach-HNSHostEndpoint -EndpointID $endpoint.ID -CompartmentID 1 +} + +# Return HNS endpoint IP +(Get-NetIPConfiguration -AllCompartments -All -Detailed | where { $_.NetAdapter.LinkLayerAddress -eq $endpoint.MacAddress }).IPV4Address.IPAddress.Trim() diff --git a/pkg/nodeconfig/nodeconfig.go b/pkg/nodeconfig/nodeconfig.go index 338bf3e581..4d631c5a0e 100644 --- a/pkg/nodeconfig/nodeconfig.go +++ b/pkg/nodeconfig/nodeconfig.go @@ -345,9 +345,6 @@ func (nc *nodeConfig) configureNetwork() error { } // Configure CNI in the Windows VM - if err := nc.configureCNI(); err != nil { - return errors.Wrapf(err, "error configuring CNI for %s", nc.node.GetName()) - } // Start the kube-proxy service if err := nc.Windows.ConfigureKubeProxy(nc.node.GetName(), nc.node.Annotations[HybridOverlaySubnet]); err != nil { return errors.Wrapf(err, "error starting kube-proxy for %s", nc.node.GetName()) @@ -421,7 +418,7 @@ func (nc *nodeConfig) configureCNI() error { return errors.Wrap(err, "error populating host subnet in node network") } // populate the CNI config file with the host subnet, service network CIDR and IP address of the Windows VM - configFile, err := nc.network.populateCniConfig(nc.clusterServiceCIDR, nc.GetIPv4Address(), payload.CNIConfigTemplatePath) + configFile, err := nc.network.populateCniConfig(nc.clusterServiceCIDR, nc.GetIPv4Address(), payload.CNIConfigScriptTemplatePath) if err != nil { return errors.Wrapf(err, "error populating CNI config file %s", configFile) } diff --git a/pkg/nodeconfig/payload/payload.go b/pkg/nodeconfig/payload/payload.go index c0269eb5b9..f6c5a05476 100644 --- a/pkg/nodeconfig/payload/payload.go +++ b/pkg/nodeconfig/payload/payload.go @@ -3,9 +3,8 @@ package payload import ( "crypto/sha256" "fmt" - "io/ioutil" - "github.com/pkg/errors" + "io/ioutil" ) // Payload files @@ -46,8 +45,10 @@ const ( // WinOverlayCNIPlugin is the path of the win-overlay CNI Plugin binary. The container image should already have // this binary mounted WinOverlayCNIPlugin = payloadDirectory + cniDirectory + "win-overlay.exe" - // CNIConfigTemplatePath is the path for CNI config template - CNIConfigTemplatePath = payloadDirectory + cniDirectory + "cni-conf-template.json" + // CNIConfigScriptTemplatePath is the path for CNI config template + CNIConfigScriptTemplatePath = payloadDirectory + "/powershell/cni-conf-template.ps1" + // CNIConfigScriptPath is the path for generated CNI Config Script + CNIConfigScriptPath = payloadDirectory + "/generated/cni-conf.ps1" // HybridOverlayName is the name of the hybrid overlay executable HybridOverlayName = "hybrid-overlay-node.exe" // HybridOverlayPath contains the path of the hybrid overlay binary. The container image should already have this diff --git a/pkg/services/services.go b/pkg/services/services.go index 0b2f15763a..7ca8fcb242 100644 --- a/pkg/services/services.go +++ b/pkg/services/services.go @@ -2,6 +2,7 @@ package services import ( "fmt" + "github.com/openshift/windows-machine-config-operator/pkg/nodeconfig" "github.com/openshift/windows-machine-config-operator/pkg/servicescm" "github.com/openshift/windows-machine-config-operator/pkg/windows" @@ -20,6 +21,7 @@ func GenerateManifest(vxlanPort string, debug bool) (*servicescm.Data, error) { Priority: 1, }, hybridOverlayConfiguration(vxlanPort, debug), + kubeProxyConfiguration(), } // TODO: All payload filenames and checksums must be added here https://issues.redhat.com/browse/WINC-847 files := &[]servicescm.FileInfo{} @@ -56,3 +58,36 @@ func hybridOverlayConfiguration(vxlanPort string, debug bool) servicescm.Service Priority: 1, } } + +// kubeProxyConfiguration returns the Service definition for kube-proxy +func kubeProxyConfiguration() servicescm.Service { + kubeProxyServiceCommand := fmt.Sprintf("%s --windows-service --v=4 --proxy-mode=kernelspace "+ + "--feature-gates=WinOverlay=true --hostname-override=NODE_NAME --kubeconfig=%s --cluster-cidr=NODE_SUBNET "+ + "--log-dir=%s --logtostderr=false --network-name=%s --source-vip=SUBNET --enable-dsr=false", + windows.KubeProxyPath, windows.KubeconfigPath, windows.KubeProxyLogDir, + windows.OVNKubeOverlayNetwork) + + return servicescm.Service{ + Name: windows.KubeProxyServiceName, + Command: kubeProxyServiceCommand, + NodeVariablesInCommand: []servicescm.NodeCmdArg{ + { + Name: "NODE_NAME", + NodeObjectJsonPath: "{.metadata.name}", + }, + { + Name: "NODE_SUBNET", + NodeObjectJsonPath: fmt.Sprintf("{.metadata.annotations[%s]}", nodeconfig.HybridOverlaySubnet), + }, + }, + PowershellVariablesInCommand: []servicescm.PowershellCmdArg{ + { + Name: "SOURCE_VIP", + Path: "", + }, + }, + Dependencies: []string{windows.HybridOverlayServiceName}, + Bootstrap: false, + Priority: 2, + } +} diff --git a/pkg/windows/windows.go b/pkg/windows/windows.go index 411ec0611c..d7c1582eab 100644 --- a/pkg/windows/windows.go +++ b/pkg/windows/windows.go @@ -27,16 +27,16 @@ const ( winTemp = "C:\\Windows\\Temp\\" // wgetIgnoreCertCmd is the remote location of the wget-ignore-cert.ps1 script wgetIgnoreCertCmd = remoteDir + "wget-ignore-cert.ps1" - // hnsPSModule is the remote location of the hns.psm1 module - hnsPSModule = remoteDir + "hns.psm1" + // HNSPSModule is the remote location of the hns.psm1 module + HNSPSModule = remoteDir + "hns.psm1" // k8sDir is the remote kubernetes executable directory k8sDir = "C:\\k\\" //KubeconfigPath is the remote location of the kubelet's kubeconfig KubeconfigPath = k8sDir + "kubeconfig" // logDir is the remote kubernetes log directory logDir = "C:\\var\\log\\" - // kubeProxyLogDir is the remote kube-proxy log directory - kubeProxyLogDir = logDir + "kube-proxy\\" + // KubeProxyLogDir is the remote kube-proxy log directory + KubeProxyLogDir = logDir + "kube-proxy\\" // HybridOverlayLogDir is the remote hybrid-overlay log directory HybridOverlayLogDir = logDir + "hybrid-overlay\\" // ContainerdLogDir is the remote containerd log directory @@ -45,8 +45,8 @@ const ( wicdLogDir = logDir + "wicd\\" // cniDir is the directory for storing CNI binaries cniDir = k8sDir + "cni\\" - // cniConfDir is the directory for storing CNI configuration - cniConfDir = cniDir + "config\\" + // CniConfDir is the directory for storing CNI configuration + CniConfDir = cniDir + "config\\" // ContainerdDir is the directory for storing Containerd binary ContainerdDir = k8sDir + "containerd\\" // containerdPath is the location of the containerd exe @@ -61,8 +61,8 @@ const ( windowsExporterPath = k8sDir + "windows_exporter.exe" // azureCloudNodeManagerPath is the location of the azure-cloud-node-manager.exe azureCloudNodeManagerPath = k8sDir + payload.AzureCloudNodeManager - // kubeProxyPath is the location of the kube-proxy exe - kubeProxyPath = k8sDir + "kube-proxy.exe" + // KubeProxyPath is the location of the kube-proxy exe + KubeProxyPath = k8sDir + "kube-proxy.exe" // HybridOverlayPath is the location of the hybrid-overlay-node exe HybridOverlayPath = k8sDir + "hybrid-overlay-node.exe" // HybridOverlayServiceName is the name of the hybrid-overlay-node Windows service @@ -71,8 +71,8 @@ const ( BaseOVNKubeOverlayNetwork = "BaseOVNKubernetesHybridOverlayNetwork" // OVNKubeOverlayNetwork is the name of the OVN HNS Overlay network OVNKubeOverlayNetwork = "OVNKubernetesHybridOverlayNetwork" - // kubeProxyServiceName is the name of the kube-proxy Windows service - kubeProxyServiceName = "kube-proxy" + // KubeProxyServiceName is the name of the kube-proxy Windows service + KubeProxyServiceName = "kube-proxy" // KubeletServiceName is the name of the kubelet Windows service KubeletServiceName = "kubelet" // WindowsExporterServiceName is the name of the windows_exporter Windows service @@ -99,6 +99,7 @@ const ( ManagedTag = "OpenShift managed" // containersFeatureName is the name of the Windows feature that is required to be enabled on the Windows instance. containersFeatureName = "Containers" + CNIConfScript = remoteDir + "cni-conf.ps1" ) var ( @@ -109,21 +110,21 @@ var ( // dependent service should be placed before the service it depends on. RequiredServices = []string{ WindowsExporterServiceName, - kubeProxyServiceName, + KubeProxyServiceName, HybridOverlayServiceName, KubeletServiceName, wicdServiceName, containerdServiceName} // RequiredServicesOwnedByWICD is the list of services owned by WICD which should be running on all Windows nodes. - RequiredServicesOwnedByWICD = []string{WindowsExporterServiceName, HybridOverlayServiceName} + RequiredServicesOwnedByWICD = []string{WindowsExporterServiceName, HybridOverlayServiceName, KubeProxyServiceName} // RequiredDirectories is a list of directories to be created by WMCO RequiredDirectories = []string{ k8sDir, remoteDir, cniDir, - cniConfDir, + CniConfDir, logDir, - kubeProxyLogDir, + KubeProxyLogDir, wicdLogDir, HybridOverlayLogDir, ContainerdDir, @@ -151,6 +152,7 @@ func getFilesToTransfer() (map[*payload.FileInfo]string, error) { payload.ContainerdPath: ContainerdDir, payload.HcsshimPath: ContainerdDir, payload.ContainerdConfPath: ContainerdDir, + payload.CNIConfigScriptPath: remoteDir, } files := make(map[*payload.FileInfo]string) for src, dest := range srcDestPairs { @@ -521,8 +523,8 @@ func (vm *windows) EnsureCNIConfig(configFile string) error { if err != nil { return errors.Wrap(err, "unable to get info for the CNI config file") } - if err := vm.EnsureFile(file, cniConfDir); err != nil { - return errors.Errorf("unable to copy CNI file %s to %s", configFile, cniConfDir) + if err := vm.EnsureFile(file, CniConfDir); err != nil { + return errors.Errorf("unable to copy CNI file %s to %s", configFile, CniConfDir) } return nil } @@ -547,27 +549,27 @@ func (vm *windows) ConfigureAzureCloudNodeManager(nodeName string) error { } func (vm *windows) ConfigureKubeProxy(nodeName, hostSubnet string) error { - endpointIP, err := vm.createHNSEndpoint() + endpointIP, err := vm.Run(CNIConfScript, true) if err != nil { return errors.Wrap(err, "error creating HNS endpoint") } kubeProxyServiceArgs := "--windows-service --v=4 --proxy-mode=kernelspace --feature-gates=WinOverlay=true " + "--hostname-override=" + nodeName + " --kubeconfig=c:\\k\\kubeconfig " + - "--cluster-cidr=" + hostSubnet + " --log-dir=" + kubeProxyLogDir + " --logtostderr=false " + - "--network-name=" + OVNKubeOverlayNetwork + " --source-vip=" + endpointIP + + "--cluster-cidr=" + hostSubnet + " --log-dir=" + KubeProxyLogDir + " --logtostderr=false " + + "--network-name=" + OVNKubeOverlayNetwork + " --source-vip=" + strings.TrimSpace(endpointIP) + " --enable-dsr=false" - kubeProxyService, err := newService(kubeProxyPath, kubeProxyServiceName, kubeProxyServiceArgs, + kubeProxyService, err := newService(KubeProxyPath, KubeProxyServiceName, kubeProxyServiceArgs, []string{HybridOverlayServiceName}) if err != nil { - return errors.Wrapf(err, "error creating %s service object", kubeProxyServiceName) + return errors.Wrapf(err, "error creating %s service object", KubeProxyServiceName) } if err := vm.ensureServiceIsRunning(kubeProxyService); err != nil { - return errors.Wrapf(err, "error ensuring %s Windows service has started running", kubeProxyServiceName) + return errors.Wrapf(err, "error ensuring %s Windows service has started running", KubeProxyServiceName) } - vm.log.Info("configured", "service", kubeProxyServiceName, "args", kubeProxyServiceArgs) + vm.log.Info("configured", "service", KubeProxyServiceName, "args", kubeProxyServiceArgs) return nil } @@ -929,7 +931,7 @@ func (vm *windows) waitForServiceToRun(serviceName string) error { // createHNSEndpoint makes a request to create a new HNS endpoint for the Hybrid Overlay network on the VM. On success, // the IP of the endpoint will be returned. func (vm *windows) createHNSEndpoint() (string, error) { - cmd := "Import-Module -DisableNameChecking " + hnsPSModule + "; " + + cmd := "Import-Module -DisableNameChecking " + HNSPSModule + "; " + "$net = (Get-HnsNetwork | where { $_.Name -eq '" + OVNKubeOverlayNetwork + "' }); " + "$endpoint = New-HnsEndpoint -NetworkId $net.ID -Name VIPEndpoint; " + "Attach-HNSHostEndpoint -EndpointID $endpoint.ID -CompartmentID 1; " +