Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use exponential backoff for DNS updates #10996

Merged
merged 1 commit into from
Mar 9, 2021
Merged

Use exponential backoff for DNS updates #10996

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 16 additions & 4 deletions dns-controller/pkg/dns/dnscontroller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
var zoneListCacheValidity = time.Minute * 15

const DefaultTTL = time.Minute
const MaxFailures = 5

// DNSController applies the desired DNS state to the DNS backend
type DNSController struct {
Expand All @@ -56,6 +57,8 @@ type DNSController struct {
// changeCount is a change-counter, which helps us avoid computation when nothing has changed
changeCount uint64

// failCount is a fail-counter for exponential backoff, reset on success
failCount uint64
// update loop frequency (seconds)
updateInterval time.Duration
}
Expand Down Expand Up @@ -120,9 +123,18 @@ func (c *DNSController) runWatcher(stopCh <-chan struct{}) {
}

if err != nil {
klog.Warningf("Unexpected error in DNS controller, will retry: %v", err)
time.Sleep(2 * c.updateInterval)
// Increment the update failure counter
failures := atomic.AddUint64(&c.failCount, 1)
// Avoid overflowing the exponential backoff interval
if failures > MaxFailures {
failures = MaxFailures
}
backoffInterval := 1 << failures * c.updateInterval
klog.Warningf("Unexpected error in DNS controller, will retry in %s: %v", backoffInterval, err)
time.Sleep(backoffInterval)
} else {
// Reset the update failure counter
atomic.StoreUint64(&c.failCount, 0)
// Simple debouncing; DNS servers are typically pretty slow anyway
time.Sleep(c.updateInterval)
}
Expand Down Expand Up @@ -308,7 +320,7 @@ func (c *DNSController) runOnce() error {
continue
}

klog.V(2).Infof("applying DNS changeset for zone %s", key)
klog.V(2).Infof("Applying DNS changeset for zone %s", key)
if err := changeset.Apply(ctx); err != nil {
klog.Warningf("error applying DNS changeset for zone %s: %v", key, err)
errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err))
Expand Down Expand Up @@ -351,7 +363,7 @@ func (c *DNSController) RemoveRecordsImmediate(records []Record) error {
}

for key, changeset := range op.changesets {
klog.V(2).Infof("applying DNS changeset for zone %s", key)
klog.V(2).Infof("Applying DNS changeset for zone %s", key)
if err := changeset.Apply(ctx); err != nil {
klog.Warningf("error applying DNS changeset for zone %s: %v", key, err)
errors = append(errors, fmt.Errorf("error applying DNS changeset for zone %s: %v", key, err))
Expand Down