Skip to content

Commit

Permalink
Merge pull request #769 from nilo19/feat/inotify
Browse files Browse the repository at this point in the history
feat: support reloading the cloud controller manager based on the changes of the config file
  • Loading branch information
k8s-ci-robot committed Aug 25, 2021
2 parents 5db2bdc + 36c1635 commit 370810d
Show file tree
Hide file tree
Showing 5 changed files with 116 additions and 18 deletions.
29 changes: 17 additions & 12 deletions cmd/cloud-controller-manager/app/controllermanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,21 @@ func RunWrapper(s *options.CloudControllerManagerOptions, c *cloudcontrollerconf

panic("unreachable")
} else {
klog.V(1).Infof("RunWrapper: using dynamic initialization from secret %s/%s, starting the secret watcher", c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigSecretName)
updateCh := dynamic.RunSecretWatcherOrDie(c)
errCh := make(chan error, 1)
var updateCh chan struct{}
if c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile != "" {
klog.V(1).Infof("RunWrapper: using dynamic initialization from config file %s, starting the file watcher", c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
updateCh = dynamic.RunFileWatcherOrDie(c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile)
} else {
klog.V(1).Infof("RunWrapper: using dynamic initialization from secret %s/%s, starting the secret watcher", c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigSecretName)
updateCh = dynamic.RunSecretWatcherOrDie(c)
}

errCh := make(chan error, 1)
cancelFunc := runAsync(s, errCh)

for {
select {
case <-updateCh:
klog.V(2).Infof("RunWrapper: detected the cloud config secret %s/%s has been updated, re-constructing the cloud controller manager", c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigSecretName)
klog.V(2).Info("RunWrapper: detected the cloud config has been updated, re-constructing the cloud controller manager")

// stop the previous goroutines
cancelFunc()
Expand Down Expand Up @@ -260,20 +265,20 @@ func Run(ctx context.Context, c *cloudcontrollerconfig.CompletedConfig) error {
err error
)

if c.DynamicReloadingConfig.EnableDynamicReloading {
cloud, err = provider.NewCloudFromSecret(c.ClientBuilder, c.DynamicReloadingConfig.CloudConfigSecretName, c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigKey)
if err != nil {
klog.Fatalf("Run: Cloud provider azure could not be initialized dynamically from secret %s/%s: %v", c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigSecretName, err)
}
} else {
if c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile != "" {
cloud, err = provider.NewCloudFromConfigFile(c.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, true)
if err != nil {
klog.Fatalf("Cloud provider azure could not be initialized: %v", err)
}
} else if c.DynamicReloadingConfig.EnableDynamicReloading && c.DynamicReloadingConfig.CloudConfigSecretName != "" {
cloud, err = provider.NewCloudFromSecret(c.ClientBuilder, c.DynamicReloadingConfig.CloudConfigSecretName, c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigKey)
if err != nil {
klog.Fatalf("Run: Cloud provider azure could not be initialized dynamically from secret %s/%s: %v", c.DynamicReloadingConfig.CloudConfigSecretNamespace, c.DynamicReloadingConfig.CloudConfigSecretName, err)
}
}

if cloud == nil {
klog.Fatalf("cloud provider is nil")
klog.Fatalf("cloud provider is nil, please check if the --cloud-config is set properly")
}

if !cloud.HasClusterID() {
Expand Down
91 changes: 91 additions & 0 deletions cmd/cloud-controller-manager/app/dynamic/file_watcher.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package dynamic

import (
"os"
"strings"
"time"

"github.com/fsnotify/fsnotify"

"k8s.io/klog/v2"
)

func RunFileWatcherOrDie(path string) chan struct{} {
watcher, err := fsnotify.NewWatcher()
if err != nil {
klog.Errorf("RunFileWatcherOrDie: failed to initialize file watcher: %s", err)
os.Exit(1)
}

startWatchingOrDie(watcher, path, 5)

updateChan := make(chan struct{})
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
klog.Error("RunFileWatcherOrDie: events channel closed unexpectedly")
_ = watcher.Close()
os.Exit(1)
}

klog.Infof("RunFileWatcherOrDie: found file update event: %v", event)
updateChan <- struct{}{}

if strings.EqualFold(event.Name, path) && event.Op&fsnotify.Remove == fsnotify.Remove {
startWatchingOrDie(watcher, path, 5)
}
case err, ok := <-watcher.Errors:
if !ok {
klog.Error("RunFileWatcherOrDie: errors channel closed unexpectedly")
_ = watcher.Close()
os.Exit(1)
}

klog.Errorf("RunFileWatcherOrDie: failed to watch file %s: %s", path, err.Error())
_ = watcher.Close()
os.Exit(1)
}
}
}()

return updateChan
}

func startWatchingOrDie(watcher *fsnotify.Watcher, path string, maxRetries int) {
attempt := 0
for {
err := watcher.Add(path)
if err != nil {
if attempt < maxRetries {
attempt++
klog.Warningf("RunFileWatcherOrDie: failed to watch %s: %s, will retry", path, err.Error())
time.Sleep(time.Second)
continue
} else {
klog.Errorf("RunFileWatcherOrDie: failed to watch %s after %d times retry", maxRetries)
_ = watcher.Close()
os.Exit(1)
}
}

break
}
}
8 changes: 4 additions & 4 deletions cmd/cloud-controller-manager/app/options/dynamic.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ func (o *DynamicReloadingOptions) AddFlags(fs *pflag.FlagSet) {
return
}

fs.BoolVar(&o.EnableDynamicReloading, "enable-dynamic-reloading", false, "Enable re-configuring cloud controller manager from secret without restarting")
fs.StringVar(&o.CloudConfigSecretName, "cloud-config-secret-name", "azure-cloud-provider", "The name of the cloud config secret, default to 'cloud-provider-config'")
fs.StringVar(&o.CloudConfigSecretNamespace, "cloud-config-secret-namespace", "kube-system", "The k8s namespace of the cloud config secret, default to 'kube-system'")
fs.StringVar(&o.CloudConfigKey, "cloud-config-key", "cloud-config", "The key of the config data in the cloud config secret, default to 'cloud-config'")
fs.BoolVar(&o.EnableDynamicReloading, "enable-dynamic-reloading", false, "Enable re-configuring cloud controller manager from secret without restarting.")
fs.StringVar(&o.CloudConfigSecretName, "cloud-config-secret-name", "", "The name of the cloud config secret.")
fs.StringVar(&o.CloudConfigSecretNamespace, "cloud-config-secret-namespace", "kube-system", "The k8s namespace of the cloud config secret, default to 'kube-system'.")
fs.StringVar(&o.CloudConfigKey, "cloud-config-key", "cloud-config", "The key of the config data in the cloud config secret, default to 'cloud-config'.")
}

// ApplyTo fills up dynamic reloading config with options
Expand Down
6 changes: 4 additions & 2 deletions site/content/en/install/configs.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,11 +129,11 @@ Master nodes are not added to the backends of Azure Load Balancer (ALB) if `excl

By default, if nodes are labeled with `node-role.kubernetes.io/master`, they would also be excluded from ALB. If you want to add the master nodes to ALB, `excludeMasterFromStandardLB` should be set to false and label `node-role.kubernetes.io/master` should be removed if it has already been applied.

### Setting Azure cloud provider from Kubernetes secrets
### Dynamically reloading cloud controller manager

Since v1.21.0, Azure cloud provider supports reading the cloud config from Kubernetes secrets. The secret is a serialized version of `azure.json` file. When the secret is changed, the cloud controller manager will re-constructing itself without restarting the pod.

To enable this feature, set `--enable-dynamic-reloading=true` and configure the secret name, namespace and data key by `--cloud-config-secret-name`, `--cloud-config-secret-namespace` and `--cloud-config-key`. When initializing from secret, the `--cloud-config` would be ignored.
To enable this feature, set `--enable-dynamic-reloading=true` and configure the secret name, namespace and data key by `--cloud-config-secret-name`, `--cloud-config-secret-namespace` and `--cloud-config-key`. When initializing from secret, the `--cloud-config` should not be set.

> Note that the `--enable-dynamic-reloading` cannot be `false` if `--cloud-config` is empty. To build the cloud provider from classic config file, please explicitly specify the `--cloud-config` and do not set `--enable-dynamic-reloading=true`. In this manner, the cloud controller manager will not be updated when the config file is changed. You need to restart the pod to manually trigger the re-initialization.
Expand Down Expand Up @@ -170,6 +170,8 @@ subjects:
namespace: kube-system
```

It is also supported to build the cloud controller manager from the cloud config file and reload dynamically. To use this way, turn on `--enable-dynamic-reloading` and set `--cloud-config` to an non-empty value.

### per client rate limiting

Since v1.18.0, the original global rate limiting has been switched to per-client. A set of new rate limit configure options are introduced for each client, which includes:
Expand Down

0 comments on commit 370810d

Please sign in to comment.