Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Optimization of pod batch restart in cluster upgrade #667

Closed
wants to merge 2 commits into from
Closed

Optimization of pod batch restart in cluster upgrade #667

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c585716
added or modified labels not due to restart pods
xiedeyantu Mar 5, 2021
187fb6e
Optimize the statefulset update, patching labels first will not trigg…
xiedeyantu Mar 6, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions pkg/controller/chi/creator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ import (
"errors"
"fmt"
"github.com/altinity/clickhouse-operator/pkg/util"

apps "k8s.io/api/apps/v1"
"k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"

log "github.com/altinity/clickhouse-operator/pkg/announcer"
chop "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
Expand Down Expand Up @@ -51,8 +51,24 @@ func (c *Controller) createStatefulSet(statefulSet *apps.StatefulSet, host *chop
func (c *Controller) updateStatefulSet(oldStatefulSet *apps.StatefulSet, newStatefulSet *apps.StatefulSet, host *chop.ChiHost) error {
log.V(2).M(host).F().P()

// diff labels
diffLabelsBytes, err := DiffLabels(oldStatefulSet, newStatefulSet)
if err != nil {
log.V(1).M(host).A().Error("%v", err)
return err
}
// Patch labels
updatedStatefulSet, err := c.kubeClient.AppsV1().StatefulSets(newStatefulSet.Namespace).
Patch(oldStatefulSet.GetName(), types.JSONPatchType, diffLabelsBytes)
if err == nil {
log.V(2).M(host).F().Info("labels changed")
} else {
log.V(1).M(host).A().Error("%v", err)
return err
}

// Apply newStatefulSet and wait for Generation to change
updatedStatefulSet, err := c.kubeClient.AppsV1().StatefulSets(newStatefulSet.Namespace).Update(newStatefulSet)
updatedStatefulSet, err = c.kubeClient.AppsV1().StatefulSets(newStatefulSet.Namespace).Update(updatedStatefulSet)
if err != nil {
// Update failed
log.V(1).M(host).A().Error("%v", err)
Expand Down
41 changes: 41 additions & 0 deletions pkg/controller/chi/labeler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@
package chi

import (
"encoding/json"
"gopkg.in/d4l3k/messagediff.v1"
apps "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1"
"strings"

log "github.com/altinity/clickhouse-operator/pkg/announcer"
chiv1 "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1"
Expand Down Expand Up @@ -144,3 +148,40 @@ func (c *Controller) addLabels(meta *v1.ObjectMeta) {
},
)
}

type PatchStringValue struct {
Op string `json:"op"`
Path string `json:"path"`
Value string `json:"value"`
}

func DiffLabels(oldStatefulSet *apps.StatefulSet, newStatefulSet *apps.StatefulSet) ([]byte, error) {
var labels []PatchStringValue
labelsDiff, _ := messagediff.DeepDiff(oldStatefulSet.Labels, newStatefulSet.Labels)
for path, value := range labelsDiff.Added {
labels = append(labels, PatchStringValue{
Op: labelAdd,
Path: labelPath + strings.Replace(strings.Trim((*path)[0].String(), "[\"]"), "/", "~1", -1),
Value: value.(string),
})
}
for path, value := range labelsDiff.Modified {
labels = append(labels, PatchStringValue{
Op: labelReplace,
Path: labelPath + strings.Replace(strings.Trim((*path)[0].String(), "[\"]"), "/", "~1", -1),
Value: value.(string),
})
}
for path, value := range labelsDiff.Removed {
labels = append(labels, PatchStringValue{
Op: labelRemove,
Path: labelPath + strings.Replace(strings.Trim((*path)[0].String(), "[\"]"), "/", "~1", -1),
Value: value.(string),
})
}
labelsBytes, err := json.Marshal(labels)
if err != nil {
return nil, err
}
return labelsBytes, nil
}
8 changes: 8 additions & 0 deletions pkg/controller/chi/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,14 @@ const (
reconcileDelete = "delete"
)

const (
labelAdd = "add"
labelReplace = "replace"
labelRemove = "remove"
)

const labelPath = "/metadata/labels/"

type ReconcileChi struct {
cmd string
old *chi.ClickHouseInstallation
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/chi/worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,7 +253,7 @@ func (w *worker) updateCHI(old, new *chop.ClickHouseInstallation) error {
WithStatusAction(new).
M(new).F().
Info("reconcile started")
w.a.V(2).M(new).F().Info("action plan\n%s\n", actionPlan.String())
//w.a.V(2).M(new).F().Info("action plan\n%s\n", actionPlan.String())
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was it intentional to disable this log line? (If so: why not remove it completely?)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When debugging, this will report null pointer exception. I wonder if it is an environment problem, so do this.


if new.IsStopped() {
w.a.V(1).
Expand Down