Skip to content

Commit

Permalink
Fix flag passing in CCM.
Browse files Browse the repository at this point in the history
  • Loading branch information
@cici37
cici37 committed Mar 31, 2021
1 parent 7483d6f commit 6a1c8c4
Show file tree
Hide file tree
Showing 6 changed files with 110 additions and 231 deletions.
97 changes: 40 additions & 57 deletions cmd/cloud-controller-manager/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,20 @@ limitations under the License.

// This file should be written by each cloud provider.
// For an minimal working example, please refer to k8s.io/cloud-provider/sample/basic_main.go
// For an advanced example, please refer to k8s.io/cloud-provider/sample/advanced_main.go
// For more details, please refer to k8s.io/kubernetes/cmd/cloud-controller-manager/main.go
// The current file demonstrate how other cloud provider should leverage CCM and it uses fake parameters. Please modify for your own use.

package main

import (
"fmt"
"math/rand"
"net/http"
"os"
"time"

"github.com/spf13/pflag"
cloudprovider "k8s.io/cloud-provider"

"k8s.io/cloud-provider"
"k8s.io/cloud-provider/app"
cloudcontrollerconfig "k8s.io/cloud-provider/app/config"
"k8s.io/cloud-provider/options"
Expand All @@ -56,67 +55,51 @@ const (
func main() {
rand.Seed(time.Now().UnixNano())

// cloudProviderConfigFile shows an sample of parse config file from flag option
var flagset *pflag.FlagSet = pflag.NewFlagSet("flagSet", pflag.ContinueOnError)
var cloudProviderConfigFile *string = flagset.String("cloud-provider-configfile", "", "This is the sample input for cloud provider config file")
pflag.CommandLine.ParseErrorsWhitelist.UnknownFlags = true
_ = pflag.CommandLine.Parse(os.Args[1:])

// this is an example of allow-listing specific controller loops
controllerList := []string{"cloud-node", "cloud-node-lifecycle", "service", "route"}

s, err := options.NewCloudControllerManagerOptions()
ccmOptions, err := options.NewCloudControllerManagerOptions()
if err != nil {
klog.Fatalf("unable to initialize command options: %v", err)
}
c, err := s.Config(controllerList, app.ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}

cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, *cloudProviderConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("cloud provider is nil")
}
cloudInitializer := func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface {
cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile
// initialize cloud provider with the cloud provider name and config file provided
cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, cloudConfigFile)
if err != nil {
klog.Fatalf("Cloud provider could not be initialized: %v", err)
}
if cloud == nil {
klog.Fatalf("Cloud provider is nil")
}

if !cloud.HasClusterID() {
if c.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
if !cloud.HasClusterID() {
if config.ComponentConfig.KubeCloudShared.AllowUntaggedCloud {
klog.Warning("detected a cluster without a ClusterID. A ClusterID will be required in the future. Please tag your cluster to avoid any future issues")
} else {
klog.Fatalf("no ClusterID found. A ClusterID is required for the cloud provider to function properly. This check can be bypassed by setting the allow-untagged-cloud option")
}
}
}

// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(c.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(c.SharedInformers)
}
// Initialize the cloud provider with a reference to the clientBuilder
cloud.Initialize(config.ClientBuilder, make(chan struct{}))
// Set the informer on the user cloud object
if informerUserCloud, ok := cloud.(cloudprovider.InformerUser); ok {
informerUserCloud.SetInformers(config.SharedInformers)
}

controllerInitializers := app.DefaultControllerInitializers(c.Complete(), cloud)
return cloud
}

controllerInitializers := app.DefaultInitFuncConstructors
// Here is an example to remove the controller which is not needed.
// e.g. remove the cloud-node-lifecycle controller which current cloud provider does not need.
//delete(controllerInitializers, "cloud-node-lifecycle")

// Here is an example to add an controller(NodeIpamController) which will be used by cloud provider
// generate nodeipamconfig. Here is an sample code. Please pass the right parameter in your code.
// generate nodeIPAMConfig. Here is an sample code. Please pass the right parameter in your code.
// If you do not need additional controller, please ignore.
nodeipamconfig := nodeipamconfig.NodeIPAMControllerConfiguration{
ServiceCIDR: "sample",
SecondaryServiceCIDR: "sample",
NodeCIDRMaskSize: 11,
NodeCIDRMaskSizeIPv4: 11,
NodeCIDRMaskSizeIPv6: 111,
}
controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper(c.Complete(), nodeipamconfig, cloud)
controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper

command := app.NewCloudControllerManagerCommand(s, c, controllerInitializers)
command := app.NewCloudControllerManagerCommand(cloudProviderName, ccmOptions, cloudInitializer, controllerInitializers)

// TODO: once we switch everything over to Cobra commands, we can go back to calling
// utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the
Expand All @@ -127,20 +110,20 @@ func main() {
logs.InitLogs()
defer logs.FlushLogs()

// the flags could be set before execute
command.Flags().VisitAll(func(flag *pflag.Flag) {
if flag.Name == "cloud-provider" {
flag.Value.Set("SampleCloudProviderFlagValue")
return
}
})
if err := command.Execute(); err != nil {
os.Exit(1)
}
}

func startNodeIpamControllerWrapper(ccmconfig *cloudcontrollerconfig.CompletedConfig, nodeipamconfig nodeipamconfig.NodeIPAMControllerConfiguration, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startNodeIpamControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) app.InitFunc {
nodeIPAMConfig := nodeipamconfig.NodeIPAMControllerConfiguration{
ServiceCIDR: "sample",
SecondaryServiceCIDR: "sample",
NodeCIDRMaskSize: 11,
NodeCIDRMaskSizeIPv4: 11,
NodeCIDRMaskSizeIPv6: 111,
}
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startNodeIpamController(ccmconfig, nodeipamconfig, ctx, cloud)
return startNodeIpamController(completedConfig, nodeIPAMConfig, ctx, cloud)
}
}
56 changes: 40 additions & 16 deletions staging/src/k8s.io/cloud-provider/app/controllermanager.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,21 +64,35 @@ const (
)

// NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters
func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, c *cloudcontrollerconfig.Config, controllerInitializers map[string]InitFunc) *cobra.Command {
func NewCloudControllerManagerCommand(cloudProviderName string, s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor) *cobra.Command {

cmd := &cobra.Command{
Use: "cloud-controller-manager",
Long: `The Cloud controller manager is a daemon that embeds
the cloud specific control loops shipped with Kubernetes.`,
Run: func(cmd *cobra.Command, args []string) {
verflag.PrintAndExitIfRequested()

cloudProviderFlag := cmd.Flags().Lookup("cloud-provider")
if cloudProviderFlag.Value.String() == "" {
cloudProviderFlag.Value.Set(cloudProviderName)
}
cliflag.PrintFlags(cmd.Flags())

if err := Run(c.Complete(), controllerInitializers, wait.NeverStop); err != nil {
c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
if err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}

completedConfig := c.Complete()
cloud := cloudInitializer(completedConfig)
controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud)

if err := Run(completedConfig, controllerInitializers, wait.NeverStop); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
},
Args: func(cmd *cobra.Command, args []string) error {
for _, arg := range args {
Expand All @@ -91,7 +105,7 @@ the cloud specific control loops shipped with Kubernetes.`,
}

fs := cmd.Flags()
namedFlagSets := s.Flags(KnownControllers(controllerInitializers), ControllersDisabledByDefault.List())
namedFlagSets := s.Flags(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List())
verflag.AddFlags(namedFlagSets.FlagSet("global"))
globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())

Expand Down Expand Up @@ -121,6 +135,8 @@ the cloud specific control loops shipped with Kubernetes.`,
return cmd
}

type InitCloudFunc func(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface

// Run runs the ExternalCMServer. This should never exit.
func Run(c *cloudcontrollerconfig.CompletedConfig, controllerInitializers map[string]InitFunc, stopCh <-chan struct{}) error {
// To help debugging, immediately log version
Expand Down Expand Up @@ -256,54 +272,62 @@ func startControllers(ctx genericcontrollermanager.ControllerContext, c *cloudco
// The bool indicates whether the controller was enabled.
type InitFunc func(ctx genericcontrollermanager.ControllerContext) (debuggingHandler http.Handler, enabled bool, err error)

// KnownControllers indicate the default controller we are known.
func KnownControllers(controllerInitializers map[string]InitFunc) []string {
ret := sets.StringKeySet(controllerInitializers)
type InitFuncConstructor func(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc

// ControllerNames indicate the default controller we are known.
func ControllerNames(initFuncConstructors map[string]InitFuncConstructor) []string {
ret := sets.StringKeySet(initFuncConstructors)
return ret.List()
}

// ControllersDisabledByDefault is the controller disabled default when starting cloud-controller managers.
var ControllersDisabledByDefault = sets.NewString()

// DefaultControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// ConstructControllerInitializers is a private map of named controller groups (you can start more than one in an init func)
// paired to their InitFunc. This allows for structured downstream composition and subdivision.
func DefaultControllerInitializers(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
func ConstructControllerInitializers(initFuncConstructors map[string]InitFuncConstructor, completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) map[string]InitFunc {
controllers := map[string]InitFunc{}
controllers["cloud-node"] = StartCloudNodeControllerWrapper(completedConfig, cloud)
controllers["cloud-node-lifecycle"] = startCloudNodeLifecycleControllerWrapper(completedConfig, cloud)
controllers["service"] = startServiceControllerWrapper(completedConfig, cloud)
controllers["route"] = startRouteControllerWrapper(completedConfig, cloud)
for name, constructor := range initFuncConstructors {
controllers[name] = constructor(completedConfig, cloud)
}
return controllers
}

// StartCloudNodeControllerWrapper is used to take cloud cofig as input and start cloud node controller
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func StartCloudNodeControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeController(completedConfig, cloud, ctx.Stop)
}
}

// startCloudNodeLifecycleControllerWrapper is used to take cloud cofig as input and start cloud node lifecycle controller
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startCloudNodeLifecycleControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startCloudNodeLifecycleController(completedConfig, cloud, ctx.Stop)
}
}

// startServiceControllerWrapper is used to take cloud cofig as input and start service controller
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startServiceControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startServiceController(completedConfig, cloud, ctx.Stop)
}
}

// startRouteControllerWrapper is used to take cloud cofig as input and start route controller
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
func startRouteControllerWrapper(completedConfig *cloudcontrollerconfig.CompletedConfig, cloud cloudprovider.Interface) InitFunc {
return func(ctx genericcontrollermanager.ControllerContext) (http.Handler, bool, error) {
return startRouteController(completedConfig, cloud, ctx.Stop)
}
}

var DefaultInitFuncConstructors = map[string]InitFuncConstructor{
"cloud-node": StartCloudNodeControllerWrapper,
"cloud-node-lifecycle": startCloudNodeLifecycleControllerWrapper,
"service": startServiceControllerWrapper,
"route": startRouteControllerWrapper,
}

// CreateControllerContext creates a context struct containing references to resources needed by the
// controllers such as the cloud provider and clientBuilder. rootClientBuilder is only used for
// the shared-informers client and token controller.
Expand Down
2 changes: 1 addition & 1 deletion staging/src/k8s.io/cloud-provider/app/testing/testserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err

errCh := make(chan error)
go func(stopCh <-chan struct{}) {
if err := app.Run(config.Complete(), app.DefaultControllerInitializers(config.Complete(), cloud), stopCh); err != nil {
if err := app.Run(config.Complete(), app.ConstructControllerInitializers(app.DefaultInitFuncConstructors, config.Complete(), cloud), stopCh); err != nil {
errCh <- err
}
}(stopCh)
Expand Down
4 changes: 2 additions & 2 deletions staging/src/k8s.io/cloud-provider/sample/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ Begin with 1.20, all cloud providers should not copy over or vender in `k8s.io/k
## Steps cloud providers shoud follow

1. Have your external repo under k8s.io. e.g. `k8s.io/cloud-provider-<provider>`
2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minial working sample and `advanced_main.go` for advanced configuration samples.
Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for detailed samples.
2. Create `main.go` file under your external repo CCM directory. Please refer to `basic_main.go` for a minimum working sample.
Note: If you have a requirement of adding/deleting controllers within CCM, please refer to `k8s.io/kubernetes/cmd/cloud-controller-manager/main.go` for extra details.
3. Build/release CCM from your external repo. For existing cloud providers, the option to import legacy providers from `k8s.io/legacy-cloud-provider/<provider>` is still available.

## Things you should NOT do
Expand Down
Loading

0 comments on commit 6a1c8c4

Please sign in to comment.