From ee6c2fe83824e22d590b316e478c85da9239bca3 Mon Sep 17 00:00:00 2001 From: Maciej Zimnoch Date: Fri, 20 Nov 2020 13:23:06 +0100 Subject: [PATCH] util: port k8s nodeaffinity library --- pkg/util/nodeaffinity/README.md | 8 + pkg/util/nodeaffinity/node_affinity_test.go | 338 ++++++++++++++++++++ pkg/util/nodeaffinity/nodeaffinity.go | 262 +++++++++++++++ 3 files changed, 608 insertions(+) create mode 100644 pkg/util/nodeaffinity/README.md create mode 100644 pkg/util/nodeaffinity/node_affinity_test.go create mode 100644 pkg/util/nodeaffinity/nodeaffinity.go diff --git a/pkg/util/nodeaffinity/README.md b/pkg/util/nodeaffinity/README.md new file mode 100644 index 0000000000..4ee1d69d19 --- /dev/null +++ b/pkg/util/nodeaffinity/README.md @@ -0,0 +1,8 @@ +This package is copied from https://github.com/kubernetes/component-helpers/tree/master/scheduling/corev1/nodeaffinity + +Code was part of https://github.com/kubernetes/kubernetes, and was extracted to separate package on v0.20.0 beta. + +Operator and cloud providers use only stable releases, so until we migrate Operator and all dependency packages to 0.20 k8s, +this have to stay here. + + diff --git a/pkg/util/nodeaffinity/node_affinity_test.go b/pkg/util/nodeaffinity/node_affinity_test.go new file mode 100644 index 0000000000..738e3b4b69 --- /dev/null +++ b/pkg/util/nodeaffinity/node_affinity_test.go @@ -0,0 +1,338 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeaffinity + +import ( + "errors" + "reflect" + "testing" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + apierrors "k8s.io/apimachinery/pkg/util/errors" +) + +func TestNodeSelectorMatch(t *testing.T) { + tests := []struct { + name string + nodeSelector v1.NodeSelector + node *v1.Node + wantErr error + wantMatch bool + }{ + { + name: "nil node", + wantMatch: false, + }, + { + name: "invalid field selector and label selector", + nodeSelector: v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1", "host_2"}, + }}, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "label_1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_1_val"}, + }}, + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1"}, + }}, + }, + { + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "invalid key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_value"}, + }}, + }, + }}, + wantErr: apierrors.NewAggregate([]error{ + errors.New(`unexpected number of value (2) for node field selector operator "In"`), + errors.New(`invalid label key "invalid key": name part must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]')`), + }), + }, + { + name: "node matches field selector, but not labels", + nodeSelector: v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "label_1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_1_val"}, + }}, + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1"}, + }}, + }, + }}, + node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "host_1"}}, + }, + { + name: "node matches field selector and label selector", + nodeSelector: v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "label_1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_1_val"}, + }}, + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1"}, + }}, + }, + }}, + node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "host_1", Labels: map[string]string{"label_1": "label_1_val"}}}, + wantMatch: true, + }, + { + name: "second term matches", + nodeSelector: v1.NodeSelector{NodeSelectorTerms: []v1.NodeSelectorTerm{ + { + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "label_1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_1_val"}, + }}, + }, + { + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1"}, + }}, + }, + }}, + node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "host_1"}}, + wantMatch: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + nodeSelector, err := NewNodeSelector(&tt.nodeSelector) + if !reflect.DeepEqual(err, tt.wantErr) { + t.Fatalf("NewNodeSelector returned error %q, want %q", err, tt.wantErr) + } + if tt.wantErr != nil { + return + } + match := nodeSelector.Match(tt.node) + if match != tt.wantMatch { + t.Errorf("NodeSelector.Match returned %t, want %t", match, tt.wantMatch) + } + }) + } +} + +func TestPreferredSchedulingTermsScore(t *testing.T) { + tests := []struct { + name string + prefSchedTerms []v1.PreferredSchedulingTerm + node *v1.Node + wantErr error + wantScore int64 + }{ + { + name: "invalid field selector and label selector", + prefSchedTerms: []v1.PreferredSchedulingTerm{ + { + Weight: 1, + Preference: v1.NodeSelectorTerm{ + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1", "host_2"}, + }}, + }, + }, + { + Weight: 1, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "label_1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_1_val"}, + }}, + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1"}, + }}, + }, + }, + { + Weight: 1, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "invalid key", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_value"}, + }}, + }, + }, + }, + wantErr: apierrors.NewAggregate([]error{ + errors.New(`unexpected number of value (2) for node field selector operator "In"`), + errors.New(`invalid label key "invalid key": name part must consist of alphanumeric characters, '-', '_' or '.', and must start and end with an alphanumeric character (e.g. 'MyName', or 'my.name', or '123-abc', regex used for validation is '([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]')`), + }), + }, + { + name: "invalid field selector but no weight, error not reported", + prefSchedTerms: []v1.PreferredSchedulingTerm{ + { + Weight: 0, + Preference: v1.NodeSelectorTerm{ + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1", "host_2"}, + }}, + }, + }, + }, + node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "host_1"}}, + }, + { + name: "first and third term match", + prefSchedTerms: []v1.PreferredSchedulingTerm{ + { + Weight: 5, + Preference: v1.NodeSelectorTerm{ + MatchFields: []v1.NodeSelectorRequirement{{ + Key: "metadata.name", + Operator: v1.NodeSelectorOpIn, + Values: []string{"host_1"}, + }}, + }, + }, + { + Weight: 7, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "unknown_label", + Operator: v1.NodeSelectorOpIn, + Values: []string{"unknown_label_val"}, + }}, + }, + }, + { + Weight: 11, + Preference: v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{{ + Key: "label_1", + Operator: v1.NodeSelectorOpIn, + Values: []string{"label_1_val"}, + }}, + }, + }, + }, + node: &v1.Node{ObjectMeta: metav1.ObjectMeta{Name: "host_1", Labels: map[string]string{"label_1": "label_1_val"}}}, + wantScore: 16, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + prefSchedTerms, err := NewPreferredSchedulingTerms(tt.prefSchedTerms) + if !reflect.DeepEqual(err, tt.wantErr) { + t.Fatalf("NewPreferredSchedulingTerms returned error %q, want %q", err, tt.wantErr) + } + if tt.wantErr != nil { + return + } + score := prefSchedTerms.Score(tt.node) + if score != tt.wantScore { + t.Errorf("PreferredSchedulingTerms.Score returned %d, want %d", score, tt.wantScore) + } + }) + } +} + +func TestNodeSelectorRequirementsAsSelector(t *testing.T) { + matchExpressions := []v1.NodeSelectorRequirement{{ + Key: "foo", + Operator: v1.NodeSelectorOpIn, + Values: []string{"bar", "baz"}, + }} + mustParse := func(s string) labels.Selector { + out, e := labels.Parse(s) + if e != nil { + panic(e) + } + return out + } + tc := []struct { + in []v1.NodeSelectorRequirement + out labels.Selector + expectErr bool + }{ + {in: nil, out: labels.Nothing()}, + {in: []v1.NodeSelectorRequirement{}, out: labels.Nothing()}, + { + in: matchExpressions, + out: mustParse("foo in (baz,bar)"), + }, + { + in: []v1.NodeSelectorRequirement{{ + Key: "foo", + Operator: v1.NodeSelectorOpExists, + Values: []string{"bar", "baz"}, + }}, + expectErr: true, + }, + { + in: []v1.NodeSelectorRequirement{{ + Key: "foo", + Operator: v1.NodeSelectorOpGt, + Values: []string{"1"}, + }}, + out: mustParse("foo>1"), + }, + { + in: []v1.NodeSelectorRequirement{{ + Key: "bar", + Operator: v1.NodeSelectorOpLt, + Values: []string{"7"}, + }}, + out: mustParse("bar<7"), + }, + } + + for i, tc := range tc { + out, err := nodeSelectorRequirementsAsSelector(tc.in) + if err == nil && tc.expectErr { + t.Errorf("[%v]expected error but got none.", i) + } + if err != nil && !tc.expectErr { + t.Errorf("[%v]did not expect error but got: %v", i, err) + } + if !reflect.DeepEqual(out, tc.out) { + t.Errorf("[%v]expected:\n\t%+v\nbut got:\n\t%+v", i, tc.out, out) + } + } +} diff --git a/pkg/util/nodeaffinity/nodeaffinity.go b/pkg/util/nodeaffinity/nodeaffinity.go new file mode 100644 index 0000000000..efca431e61 --- /dev/null +++ b/pkg/util/nodeaffinity/nodeaffinity.go @@ -0,0 +1,262 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package nodeaffinity + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/util/errors" +) + +// NodeSelector is a runtime representation of v1.NodeSelector. +type NodeSelector struct { + lazy LazyErrorNodeSelector +} + +// LazyErrorNodeSelector is a runtime representation of v1.NodeSelector that +// only reports parse errors when no terms match. +type LazyErrorNodeSelector struct { + terms []nodeSelectorTerm +} + +// NewNodeSelector returns a NodeSelector or all parsing errors found. +func NewNodeSelector(ns *v1.NodeSelector) (*NodeSelector, error) { + lazy := NewLazyErrorNodeSelector(ns) + var errs []error + for _, term := range lazy.terms { + if term.parseErr != nil { + errs = append(errs, term.parseErr) + } + } + if len(errs) != 0 { + return nil, errors.NewAggregate(errs) + } + return &NodeSelector{lazy: *lazy}, nil +} + +// NewLazyErrorNodeSelector creates a NodeSelector that only reports parse +// errors when no terms match. +func NewLazyErrorNodeSelector(ns *v1.NodeSelector) *LazyErrorNodeSelector { + parsedTerms := make([]nodeSelectorTerm, 0, len(ns.NodeSelectorTerms)) + for _, term := range ns.NodeSelectorTerms { + // nil or empty term selects no objects + if isEmptyNodeSelectorTerm(&term) { + continue + } + parsedTerms = append(parsedTerms, newNodeSelectorTerm(&term)) + } + return &LazyErrorNodeSelector{ + terms: parsedTerms, + } +} + +// Match checks whether the node labels and fields match the selector terms, ORed; +// nil or empty term matches no objects. +func (ns *NodeSelector) Match(node *v1.Node) bool { + // parse errors are reported in NewNodeSelector. + match, _ := ns.lazy.Match(node) + return match +} + +// Match checks whether the node labels and fields match the selector terms, ORed; +// nil or empty term matches no objects. +// Parse errors are only returned if no terms matched. +func (ns *LazyErrorNodeSelector) Match(node *v1.Node) (bool, error) { + if node == nil { + return false, nil + } + nodeLabels := labels.Set(node.Labels) + nodeFields := extractNodeFields(node) + + var errs []error + for _, term := range ns.terms { + match, err := term.match(nodeLabels, nodeFields) + if err != nil { + errs = append(errs, term.parseErr) + continue + } + if match { + return true, nil + } + } + return false, errors.NewAggregate(errs) +} + +// PreferredSchedulingTerms is a runtime representation of []v1.PreferredSchedulingTerms. +type PreferredSchedulingTerms struct { + terms []preferredSchedulingTerm +} + +// NewPreferredSchedulingTerms returns a PreferredSchedulingTerms or all the parsing errors found. +// If a v1.PreferredSchedulingTerm has a 0 weight, its parsing is skipped. +func NewPreferredSchedulingTerms(terms []v1.PreferredSchedulingTerm) (*PreferredSchedulingTerms, error) { + var errs []error + parsedTerms := make([]preferredSchedulingTerm, 0, len(terms)) + for _, term := range terms { + if term.Weight == 0 || isEmptyNodeSelectorTerm(&term.Preference) { + continue + } + parsedTerm := preferredSchedulingTerm{ + nodeSelectorTerm: newNodeSelectorTerm(&term.Preference), + weight: int(term.Weight), + } + if parsedTerm.parseErr != nil { + errs = append(errs, parsedTerm.parseErr) + } else { + parsedTerms = append(parsedTerms, parsedTerm) + } + } + if len(errs) != 0 { + return nil, errors.NewAggregate(errs) + } + return &PreferredSchedulingTerms{terms: parsedTerms}, nil +} + +// Score returns a score for a Node: the sum of the weights of the terms that +// match the Node. +func (t *PreferredSchedulingTerms) Score(node *v1.Node) int64 { + var score int64 + nodeLabels := labels.Set(node.Labels) + nodeFields := extractNodeFields(node) + for _, term := range t.terms { + // parse errors are reported in NewPreferredSchedulingTerms. + if ok, _ := term.match(nodeLabels, nodeFields); ok { + score += int64(term.weight) + } + } + return score +} + +func isEmptyNodeSelectorTerm(term *v1.NodeSelectorTerm) bool { + return len(term.MatchExpressions) == 0 && len(term.MatchFields) == 0 +} + +func extractNodeFields(n *v1.Node) fields.Set { + f := make(fields.Set) + if len(n.Name) > 0 { + f["metadata.name"] = n.Name + } + return f +} + +type nodeSelectorTerm struct { + matchLabels labels.Selector + matchFields fields.Selector + parseErr error +} + +func newNodeSelectorTerm(term *v1.NodeSelectorTerm) nodeSelectorTerm { + var parsedTerm nodeSelectorTerm + if len(term.MatchExpressions) != 0 { + parsedTerm.matchLabels, parsedTerm.parseErr = nodeSelectorRequirementsAsSelector(term.MatchExpressions) + if parsedTerm.parseErr != nil { + return parsedTerm + } + } + if len(term.MatchFields) != 0 { + parsedTerm.matchFields, parsedTerm.parseErr = nodeSelectorRequirementsAsFieldSelector(term.MatchFields) + } + return parsedTerm +} + +func (t *nodeSelectorTerm) match(nodeLabels labels.Set, nodeFields fields.Set) (bool, error) { + if t.parseErr != nil { + return false, t.parseErr + } + if t.matchLabels != nil && !t.matchLabels.Matches(nodeLabels) { + return false, nil + } + if t.matchFields != nil && len(nodeFields) > 0 && !t.matchFields.Matches(nodeFields) { + return false, nil + } + return true, nil +} + +// nodeSelectorRequirementsAsSelector converts the []NodeSelectorRequirement api type into a struct that implements +// labels.Selector. +func nodeSelectorRequirementsAsSelector(nsm []v1.NodeSelectorRequirement) (labels.Selector, error) { + if len(nsm) == 0 { + return labels.Nothing(), nil + } + selector := labels.NewSelector() + for _, expr := range nsm { + var op selection.Operator + switch expr.Operator { + case v1.NodeSelectorOpIn: + op = selection.In + case v1.NodeSelectorOpNotIn: + op = selection.NotIn + case v1.NodeSelectorOpExists: + op = selection.Exists + case v1.NodeSelectorOpDoesNotExist: + op = selection.DoesNotExist + case v1.NodeSelectorOpGt: + op = selection.GreaterThan + case v1.NodeSelectorOpLt: + op = selection.LessThan + default: + return nil, fmt.Errorf("%q is not a valid node selector operator", expr.Operator) + } + r, err := labels.NewRequirement(expr.Key, op, expr.Values) + if err != nil { + return nil, err + } + selector = selector.Add(*r) + } + return selector, nil +} + +// nodeSelectorRequirementsAsFieldSelector converts the []NodeSelectorRequirement core type into a struct that implements +// fields.Selector. +func nodeSelectorRequirementsAsFieldSelector(nsr []v1.NodeSelectorRequirement) (fields.Selector, error) { + if len(nsr) == 0 { + return fields.Nothing(), nil + } + + var selectors []fields.Selector + for _, expr := range nsr { + switch expr.Operator { + case v1.NodeSelectorOpIn: + if len(expr.Values) != 1 { + return nil, fmt.Errorf("unexpected number of value (%d) for node field selector operator %q", + len(expr.Values), expr.Operator) + } + selectors = append(selectors, fields.OneTermEqualSelector(expr.Key, expr.Values[0])) + + case v1.NodeSelectorOpNotIn: + if len(expr.Values) != 1 { + return nil, fmt.Errorf("unexpected number of value (%d) for node field selector operator %q", + len(expr.Values), expr.Operator) + } + selectors = append(selectors, fields.OneTermNotEqualSelector(expr.Key, expr.Values[0])) + + default: + return nil, fmt.Errorf("%q is not a valid node field selector operator", expr.Operator) + } + } + + return fields.AndSelectors(selectors...), nil +} + +type preferredSchedulingTerm struct { + nodeSelectorTerm + weight int +}