From ebab94c722da00b6714e7e5f74a88a10d25130aa Mon Sep 17 00:00:00 2001 From: cici37 Date: Wed, 10 Feb 2021 15:00:44 -0800 Subject: [PATCH] Modify integration test to fill CCM test gap --- cmd/cloud-controller-manager/main.go | 12 +- .../cloud-provider/app/controllermanager.go | 13 +- .../cloud-provider/app/testing/testserver.go | 111 ++++++++++++------ .../cloud-provider/sample/basic_main.go | 15 +-- 4 files changed, 92 insertions(+), 59 deletions(-) diff --git a/cmd/cloud-controller-manager/main.go b/cmd/cloud-controller-manager/main.go index 227897e200423..f47e009b6ee66 100644 --- a/cmd/cloud-controller-manager/main.go +++ b/cmd/cloud-controller-manager/main.go @@ -31,6 +31,7 @@ import ( "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" cloudcontrollerconfig "k8s.io/cloud-provider/app/config" @@ -44,11 +45,6 @@ import ( // e.g. _"k8s.io/legacy-cloud-providers/" ) -const ( - // cloudProviderName shows an sample of using hard coded parameter, please edit the value for your case. - cloudProviderName = "SampleCloudProviderName" -) - func main() { rand.Seed(time.Now().UnixNano()) @@ -67,7 +63,7 @@ func main() { // If you do not need additional controller, please ignore. controllerInitializers["nodeipam"] = startNodeIpamControllerWrapper - command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers) + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, controllerInitializers, wait.NeverStop) // TODO: once we switch everything over to Cobra commands, we can go back to calling // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the @@ -84,9 +80,9 @@ func main() { } func cloudInitializer(config *cloudcontrollerconfig.CompletedConfig) cloudprovider.Interface { - cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider // initialize cloud provider with the cloud provider name and config file provided - cloud, err := cloudprovider.InitCloudProvider(cloudProviderName, cloudConfigFile) + cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { klog.Fatalf("Cloud provider could not be initialized: %v", err) } diff --git a/staging/src/k8s.io/cloud-provider/app/controllermanager.go b/staging/src/k8s.io/cloud-provider/app/controllermanager.go index 4a9683f2a42ca..17e51bd9be333 100644 --- a/staging/src/k8s.io/cloud-provider/app/controllermanager.go +++ b/staging/src/k8s.io/cloud-provider/app/controllermanager.go @@ -64,30 +64,33 @@ const ( ) // NewCloudControllerManagerCommand creates a *cobra.Command object with default parameters -func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor) *cobra.Command { +func NewCloudControllerManagerCommand(s *options.CloudControllerManagerOptions, cloudInitializer InitCloudFunc, initFuncConstructor map[string]InitFuncConstructor, stopCh <-chan struct{}) *cobra.Command { cmd := &cobra.Command{ Use: "cloud-controller-manager", Long: `The Cloud controller manager is a daemon that embeds the cloud specific control loops shipped with Kubernetes.`, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { verflag.PrintAndExitIfRequested() cliflag.PrintFlags(cmd.Flags()) c, err := s.Config(ControllerNames(initFuncConstructor), ControllersDisabledByDefault.List()) if err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) + return err + //os.Exit(1) } completedConfig := c.Complete() cloud := cloudInitializer(completedConfig) controllerInitializers := ConstructControllerInitializers(initFuncConstructor, completedConfig, cloud) - if err := Run(completedConfig, cloud, controllerInitializers, wait.NeverStop); err != nil { + if err := Run(completedConfig, cloud, controllerInitializers, stopCh); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) - os.Exit(1) + return err + //os.Exit(1) } + return nil }, Args: func(cmd *cobra.Command, args []string) error { for _, arg := range args { diff --git a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go index c5c4c3ec412d6..1a5f2669c8f82 100644 --- a/staging/src/k8s.io/cloud-provider/app/testing/testserver.go +++ b/staging/src/k8s.io/cloud-provider/app/testing/testserver.go @@ -22,17 +22,18 @@ import ( "io/ioutil" "net" "os" + "strings" "time" "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" cloudprovider "k8s.io/cloud-provider" "k8s.io/cloud-provider/app" "k8s.io/cloud-provider/app/config" "k8s.io/cloud-provider/options" - - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/kubernetes" - restclient "k8s.io/client-go/rest" + "k8s.io/component-base/cli/flag" ) // TearDownFunc is to be called to tear down a test server. @@ -42,7 +43,7 @@ type TearDownFunc func() type TestServer struct { LoopbackClientConfig *restclient.Config // Rest client config using the magic token Options *options.CloudControllerManagerOptions - Config *config.Config + Config *config.CompletedConfig TearDownFn TearDownFunc // TearDown function TmpDir string // Temp Dir used, by the apiserver } @@ -62,6 +63,8 @@ type Logger interface { // enough time to remove temporary files. func StartTestServer(t Logger, customFlags []string) (result TestServer, err error) { stopCh := make(chan struct{}) + configDoneCh := make(chan string) + var capturedConfig config.CompletedConfig tearDown := func() { close(stopCh) if len(result.TmpDir) != 0 { @@ -79,58 +82,94 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err return result, fmt.Errorf("failed to create temp dir: %v", err) } - fs := pflag.NewFlagSet("test", pflag.PanicOnError) - s, err := options.NewCloudControllerManagerOptions() if err != nil { return TestServer{}, err } - namedFlagSets := s.Flags([]string{}, []string{}) - for _, f := range namedFlagSets.FlagSets { - fs.AddFlagSet(f) - } - fs.Parse(customFlags) - if s.SecureServing.BindPort != 0 { - s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort() + + cloudInitializer := func(config *config.CompletedConfig) cloudprovider.Interface { + capturedConfig = *config + configDoneCh <- "" + close(configDoneCh) + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider + cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { - return result, fmt.Errorf("failed to create listener: %v", err) + t.Fatalf("Cloud provider could not be initialized: %v", err) } s.SecureServing.ServerCert.CertDirectory = result.TmpDir - - t.Logf("cloud-controller-manager will listen securely on port %d...", s.SecureServing.BindPort) + if cloud == nil { + t.Fatalf("Cloud provider is nil") + } + return cloud + } + command := app.NewCloudControllerManagerCommand(s, cloudInitializer, app.DefaultInitFuncConstructors, stopCh) + pflag.CommandLine.SetNormalizeFunc(flag.WordSepNormalizeFunc) + + commandArgs := []string{} + listeners := []net.Listener{} + disableInsecure := false + disableSecure := false + for _, arg := range customFlags { + if strings.HasPrefix(arg, "--secure-port=") { + if arg == "--secure-port=0" { + commandArgs = append(commandArgs, arg) + disableSecure = true + } + } else if strings.HasPrefix(arg, "--port=") { + if arg == "--port=0" { + commandArgs = append(commandArgs, arg) + disableInsecure = true + } + } else if strings.HasPrefix(arg, "--cert-dir=") { + // skip it + } else { + commandArgs = append(commandArgs, arg) + } } - if s.InsecureServing.BindPort != 0 { - s.InsecureServing.Listener, s.InsecureServing.BindPort, err = createListenerOnFreePort() + if !disableSecure { + listener, bindPort, err := createListenerOnFreePort() if err != nil { return result, fmt.Errorf("failed to create listener: %v", err) } + listeners = append(listeners, listener) + commandArgs = append(commandArgs, fmt.Sprintf("--secure-port=%d", bindPort)) + commandArgs = append(commandArgs, fmt.Sprintf("--cert-dir=%s", result.TmpDir)) - t.Logf("cloud-controller-manager will listen insecurely on port %d...", s.InsecureServing.BindPort) + t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort) } + if !disableInsecure { + listener, bindPort, err := createListenerOnFreePort() + if err != nil { + return result, fmt.Errorf("failed to create listener: %v", err) + } + listeners = append(listeners, listener) + commandArgs = append(commandArgs, fmt.Sprintf("--port=%d", bindPort)) - config, err := s.Config([]string{}, []string{}) - if err != nil { - return result, fmt.Errorf("failed to create config from options: %v", err) + t.Logf("cloud-controller-manager will listen securely on port %d...", bindPort) } - cloudconfig := config.Complete().ComponentConfig.KubeCloudShared.CloudProvider - cloud, err := cloudprovider.InitCloudProvider(cloudconfig.Name, cloudconfig.CloudConfigFile) - if err != nil { - return result, fmt.Errorf("Cloud provider could not be initialized: %v", err) - } - if cloud == nil { - return result, fmt.Errorf("cloud provider is nil") + for _, listener := range listeners { + listener.Close() } errCh := make(chan error) - go func(stopCh <-chan struct{}) { - if err := app.Run(config.Complete(), cloud, app.ConstructControllerInitializers(app.DefaultInitFuncConstructors, config.Complete(), cloud), stopCh); err != nil { + go func() { + command.SetArgs(commandArgs) + if err := command.Execute(); err != nil { errCh <- err } - }(stopCh) + close(errCh) + }() + + select { + case <-configDoneCh: + + case err := <-errCh: + return result, err + } t.Logf("Waiting for /healthz to be ok...") - client, err := kubernetes.NewForConfig(config.LoopbackClientConfig) + client, err := kubernetes.NewForConfig(capturedConfig.LoopbackClientConfig) if err != nil { return result, fmt.Errorf("failed to create a client: %v", err) } @@ -154,9 +193,9 @@ func StartTestServer(t Logger, customFlags []string) (result TestServer, err err } // from here the caller must call tearDown - result.LoopbackClientConfig = config.LoopbackClientConfig + result.LoopbackClientConfig = capturedConfig.LoopbackClientConfig result.Options = s - result.Config = config + result.Config = &capturedConfig result.TearDownFn = tearDown return result, nil diff --git a/staging/src/k8s.io/cloud-provider/sample/basic_main.go b/staging/src/k8s.io/cloud-provider/sample/basic_main.go index 573f1fd6b35ea..e5749ae81218c 100644 --- a/staging/src/k8s.io/cloud-provider/sample/basic_main.go +++ b/staging/src/k8s.io/cloud-provider/sample/basic_main.go @@ -21,6 +21,7 @@ limitations under the License. package main import ( + "k8s.io/apimachinery/pkg/util/wait" "math/rand" "os" "time" @@ -39,13 +40,6 @@ import ( // e.g. _"k8s.io/legacy-cloud-providers/" ) -const ( - // The variables below are samples, please edit the value for your case. - - // sampleCloudProviderName shows an sample of using hard coded parameter for CloudProviderName - sampleCloudProviderName = "SampleCloudProviderName" -) - func main() { rand.Seed(time.Now().UnixNano()) @@ -54,7 +48,7 @@ func main() { klog.Fatalf("unable to initialize command options: %v", err) } - command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors) + command := app.NewCloudControllerManagerCommand(ccmOptions, cloudInitializer, app.DefaultInitFuncConstructors, wait.NeverStop) // TODO: once we switch everything over to Cobra commands, we can go back to calling // utilflag.InitFlags() (by removing its pflag.Parse() call). For now, we have to set the @@ -71,9 +65,10 @@ func main() { } func cloudInitializer(config *config.CompletedConfig) cloudprovider.Interface { - cloudConfigFile := config.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile + cloudConfig := config.ComponentConfig.KubeCloudShared.CloudProvider + // initialize cloud provider with the cloud provider name and config file provided - cloud, err := cloudprovider.InitCloudProvider(sampleCloudProviderName, cloudConfigFile) + cloud, err := cloudprovider.InitCloudProvider(cloudConfig.Name, cloudConfig.CloudConfigFile) if err != nil { klog.Fatalf("Cloud provider could not be initialized: %v", err) }