Separate example func and add README.md

cmd/cloud-controller-manager/README.md

@@ -0,0 +1,14 @@
+# cloud-controller-manager/example
+
+This directory provides an example of how to leverage CCM extension mechanism.
+
+## Purpose
+
+Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, an example is included to demonstrate how to leverage CCM extension mechanism to add a controller.
+Please refer to `k8s.io/cloud-provider/sample` if you do not have the requirement of adding/deleting controllers in CCM.
+
+## Things you should NOT do
+
+1. Vendor in `k8s.io/cmd/cloud-controller-manager`.
+2. Directly modify anything under `k8s.io/cmd/cloud-controller-manager` in this repo. 
+3. Make specific cloud provider changes here.

cmd/cloud-controller-manager/main.go

@@ -26,7 +26,6 @@ package main
 
 import (
 	"math/rand"
-	"net/http"
 	"os"
 	"time"
 
@@ -40,10 +39,7 @@ import (
 	"k8s.io/component-base/logs"
 	_ "k8s.io/component-base/metrics/prometheus/clientgo" // load all the prometheus client-go plugins
 	_ "k8s.io/component-base/metrics/prometheus/version"  // for version metric registration
-	genericcontrollermanager "k8s.io/controller-manager/app"
 	"k8s.io/klog/v2"
-	nodeipamcontrolleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
-	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
 	// For existing cloud providers, the option to import legacy providers is still available.
 	// e.g. _"k8s.io/legacy-cloud-providers/<provider>"
 )
@@ -115,19 +111,3 @@ func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovid
 
 	return cloud
 }
-
-func startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
-	fs := pflag.NewFlagSet("fs", pflag.ContinueOnError)
-	var nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions
-	nodeIPAMControllerOptions.AddFlags(fs)
-	errors := nodeIPAMControllerOptions.Validate()
-	if len(errors) > 0 {
-		klog.Fatal("NodeIPAM controller values are not properly.")
-	}
-	var nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration
-	nodeIPAMControllerOptions.ApplyTo(&nodeIPAMConfig)
-
-	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
-		return startNodeIpamController(completedConfig, nodeIPAMConfig, ctx, cloud)
-	}
-}

cmd/cloud-controller-manager/nodeipamcontroller.go

@@ -22,6 +22,8 @@ package main
 import (
 	"errors"
 	"fmt"
+	"github.com/spf13/pflag"
+
 	"net"
 	"net/http"
 	"strings"
@@ -31,7 +33,9 @@ import (
 	"k8s.io/cloud-provider/app"
 	cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
 	genericcontrollermanager "k8s.io/controller-manager/app"
+	"k8s.io/controller-manager/pkg/features"
 	"k8s.io/klog/v2"
+	nodeipamcontrolleroptions "k8s.io/kubernetes/cmd/kube-controller-manager/app/options"
 	nodeipamcontroller "k8s.io/kubernetes/pkg/controller/nodeipam"
 	nodeipamconfig "k8s.io/kubernetes/pkg/controller/nodeipam/config"
 	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
@@ -45,23 +49,39 @@ const (
 	defaultNodeMaskCIDRIPv6 = 64
 )
 
-func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
+func startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
+	fs := pflag.NewFlagSet("fs", pflag.ContinueOnError)
+	var nodeIPAMControllerOptions nodeipamcontrolleroptions.NodeIPAMControllerOptions
+	nodeIPAMControllerOptions.AddFlags(fs)
+	errors := nodeIPAMControllerOptions.Validate()
+	if len(errors) > 0 {
+		klog.Fatal("NodeIPAM controller values are not properly.")
+	}
+	var nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration
+	nodeIPAMControllerOptions.ApplyTo(&nodeIPAMConfig)
+
+	return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
+		return startNodeIpamController(completedConfig, nodeIPAMConfig, ctx, cloud)
+	}
+}
+
+func startNodeIpamController(ccmConfig *cloudcontrollerconfig.CompletedConfig, nodeIPAMConfig nodeipamconfig.NodeIPAMControllerConfiguration, ctx genericcontrollermanager.ControllerContext, cloud cloudprovider.Interface) (http.Handler, bool, error) {
 	var serviceCIDR *net.IPNet
 	var secondaryServiceCIDR *net.IPNet
 
 	// should we start nodeIPAM
-	if !ccmconfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
+	if !ccmConfig.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs {
 		return nil, false, nil
 	}
 
 	// failure: bad cidrs in config
-	clusterCIDRs, dualStack, err := processCIDRs(ccmconfig.ComponentConfig.KubeCloudShared.ClusterCIDR)
+	clusterCIDRs, dualStack, err := processCIDRs(ccmConfig.ComponentConfig.KubeCloudShared.ClusterCIDR)
 	if err != nil {
 		return nil, false, err
 	}
 
 	// failure: more than one cidr and dual stack is not enabled
-	if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
+	if len(clusterCIDRs) > 1 && !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
 		return nil, false, fmt.Errorf("len of ClusterCIDRs==%v and dualstack feature is not enabled", len(clusterCIDRs))
 	}
 
@@ -76,24 +96,24 @@ func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, n
 	}
 
 	// service cidr processing
-	if len(strings.TrimSpace(nodeipamconfig.ServiceCIDR)) != 0 {
-		_, serviceCIDR, err = net.ParseCIDR(nodeipamconfig.ServiceCIDR)
+	if len(strings.TrimSpace(nodeIPAMConfig.ServiceCIDR)) != 0 {
+		_, serviceCIDR, err = net.ParseCIDR(nodeIPAMConfig.ServiceCIDR)
 		if err != nil {
-			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.ServiceCIDR, err)
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeIPAMConfig.ServiceCIDR, err)
 		}
 	}
 
-	if len(strings.TrimSpace(nodeipamconfig.SecondaryServiceCIDR)) != 0 {
-		_, secondaryServiceCIDR, err = net.ParseCIDR(nodeipamconfig.SecondaryServiceCIDR)
+	if len(strings.TrimSpace(nodeIPAMConfig.SecondaryServiceCIDR)) != 0 {
+		_, secondaryServiceCIDR, err = net.ParseCIDR(nodeIPAMConfig.SecondaryServiceCIDR)
 		if err != nil {
-			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeipamconfig.SecondaryServiceCIDR, err)
+			klog.Warningf("Unsuccessful parsing of service CIDR %v: %v", nodeIPAMConfig.SecondaryServiceCIDR, err)
 		}
 	}
 
 	// the following checks are triggered if both serviceCIDR and secondaryServiceCIDR are provided
 	if serviceCIDR != nil && secondaryServiceCIDR != nil {
 		// should have dual stack flag enabled
-		if !utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
 			return nil, false, fmt.Errorf("secondary service cidr is provided and IPv6DualStack feature is not enabled")
 		}
 
@@ -108,14 +128,14 @@ func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, n
 	}
 
 	var nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6 int
-	if utilfeature.DefaultFeatureGate.Enabled(app.IPv6DualStack) {
+	if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) {
 		// only --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 supported with dual stack clusters.
 		// --node-cidr-mask-size flag is incompatible with dual stack clusters.
-		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(nodeipamconfig)
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizesDualStack(nodeIPAMConfig)
 	} else {
 		// only --node-cidr-mask-size supported with single stack clusters.
 		// --node-cidr-mask-size-ipv4 and --node-cidr-mask-size-ipv6 flags are incompatible with dual stack clusters.
-		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(nodeipamconfig)
+		nodeCIDRMaskSizeIPv4, nodeCIDRMaskSizeIPv6, err = setNodeCIDRMaskSizes(nodeIPAMConfig)
 	}
 
 	if err != nil {
@@ -133,7 +153,7 @@ func startNodeIpamController(ccmconfig *cloudcontrollerconfig.CompletedConfig, n
 		serviceCIDR,
 		secondaryServiceCIDR,
 		nodeCIDRMaskSizes,
-		ipam.CIDRAllocatorType(ccmconfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
+		ipam.CIDRAllocatorType(ccmConfig.ComponentConfig.KubeCloudShared.CIDRAllocatorType),
 	)
 	if err != nil {
 		return nil, true, err

staging/src/k8s.io/cloud-provider/sample/README.md

@@ -4,9 +4,9 @@ This directory provides sample code about how all cloud providers should leverag
 
 ## Purpose
 
-Begin with 1.20, all cloud providers should not copy over or vender in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager. 
+Begin with 1.20, all cloud providers should not copy over or vendor in `k8s.io/kubernetes/cmd/cloud-controller-manager`. Inside this directory, some sample code will be provided to demonstrate how cloud providers should leverage cloud-controller-manager. 
 
-## Steps cloud providers shoud follow
+## Steps cloud providers should follow
 
 1. Have your external repo under k8s.io. e.g. `k8s.io/cloud-provider-<provider>`
 2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minimum working sample.