Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add hostip and protocol to the hostport predicates #52421

Merged
merged 2 commits into from
Nov 15, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions plugin/pkg/scheduler/algorithm/predicates/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ type predicateMetadata struct {
pod *v1.Pod
podBestEffort bool
podRequest *schedulercache.Resource
podPorts map[int]bool
podPorts map[string]bool
//key is a pod full name with the anti-affinity rules.
matchingAntiAffinityTerms map[string][]matchingPodAntiAffinityTerm
serviceAffinityInUse bool
Expand Down Expand Up @@ -172,7 +172,7 @@ func (meta *predicateMetadata) ShallowCopy() algorithm.PredicateMetadata {
podRequest: meta.podRequest,
serviceAffinityInUse: meta.serviceAffinityInUse,
}
newPredMeta.podPorts = map[int]bool{}
newPredMeta.podPorts = map[string]bool{}
for k, v := range meta.podPorts {
newPredMeta.podPorts[k] = v
}
Expand Down
2 changes: 1 addition & 1 deletion plugin/pkg/scheduler/algorithm/predicates/metadata_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -373,7 +373,7 @@ func TestPredicateMetadata_ShallowCopy(t *testing.T) {
Memory: 300,
AllowedPodNumber: 4,
},
podPorts: map[int]bool{1234: true, 456: false},
podPorts: map[string]bool{"1234": true, "456": false},
matchingAntiAffinityTerms: map[string][]matchingPodAntiAffinityTerm{
"term1": {
{
Expand Down
11 changes: 6 additions & 5 deletions plugin/pkg/scheduler/algorithm/predicates/predicates.go
Original file line number Diff line number Diff line change
Expand Up @@ -890,7 +890,7 @@ func (s *ServiceAffinity) checkServiceAffinity(pod *v1.Pod, meta algorithm.Predi

// PodFitsHostPorts checks if a node has free ports for the requested pod ports.
func PodFitsHostPorts(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *schedulercache.NodeInfo) (bool, []algorithm.PredicateFailureReason, error) {
var wantPorts map[int]bool
var wantPorts map[string]bool
if predicateMeta, ok := meta.(*predicateMetadata); ok {
wantPorts = predicateMeta.podPorts
} else {
Expand All @@ -902,11 +902,12 @@ func PodFitsHostPorts(pod *v1.Pod, meta algorithm.PredicateMetadata, nodeInfo *s
}

existingPorts := nodeInfo.UsedPorts()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can not remember why we can not just delete the port from map, can you help to check it?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't quite sure about it either, you wrote it in #42524

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@k82cn I can open another pr to change the logic, I think delete the port from map is better

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

that's ok to me.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this problem is fixed by #54538, so no longer a problem

for wport := range wantPorts {
if wport != 0 && existingPorts[wport] {
return false, []algorithm.PredicateFailureReason{ErrPodNotFitsHostPorts}, nil
}

// try to see whether existingPorts and wantPorts will conflict or not
if portsConflict(existingPorts, wantPorts) {
return false, []algorithm.PredicateFailureReason{ErrPodNotFitsHostPorts}, nil
}

return true, nil, nil
}

Expand Down
112 changes: 87 additions & 25 deletions plugin/pkg/scheduler/algorithm/predicates/predicates_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -554,10 +554,17 @@ func TestPodFitsHost(t *testing.T) {
}
}

func newPod(host string, hostPorts ...int) *v1.Pod {
func newPod(host string, hostPortInfos ...string) *v1.Pod {
networkPorts := []v1.ContainerPort{}
for _, port := range hostPorts {
networkPorts = append(networkPorts, v1.ContainerPort{HostPort: int32(port)})
for _, portInfo := range hostPortInfos {
hostPortInfo := decode(portInfo)
hostPort, _ := strconv.Atoi(hostPortInfo.hostPort)

networkPorts = append(networkPorts, v1.ContainerPort{
HostIP: hostPortInfo.hostIP,
HostPort: int32(hostPort),
Protocol: v1.Protocol(hostPortInfo.protocol),
})
}
return &v1.Pod{
Spec: v1.PodSpec{
Expand Down Expand Up @@ -585,32 +592,88 @@ func TestPodFitsHostPorts(t *testing.T) {
test: "nothing running",
},
{
pod: newPod("m1", 8080),
pod: newPod("m1", "UDP/127.0.0.1/8080"),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please also use encode in test.

nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", 9090)),
newPod("m1", "UDP/127.0.0.1/9090")),
fits: true,
test: "other port",
},
{
pod: newPod("m1", 8080),
pod: newPod("m1", "UDP/127.0.0.1/8080"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "UDP/127.0.0.1/8080")),
fits: false,
test: "same udp port",
},
{
pod: newPod("m1", "TCP/127.0.0.1/8080"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "TCP/127.0.0.1/8080")),
fits: false,
test: "same tcp port",
},
{
pod: newPod("m1", "TCP/127.0.0.1/8080"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "TCP/127.0.0.2/8080")),
fits: true,
test: "different host ip",
},
{
pod: newPod("m1", "UDP/127.0.0.1/8080"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "TCP/127.0.0.1/8080")),
fits: true,
test: "different protocol",
},
{
pod: newPod("m1", "UDP/127.0.0.1/8000", "UDP/127.0.0.1/8080"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "UDP/127.0.0.1/8080")),
fits: false,
test: "second udp port conflict",
},
{
pod: newPod("m1", "TCP/127.0.0.1/8001", "UDP/127.0.0.1/8080"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "TCP/127.0.0.1/8001", "UDP/127.0.0.1/8081")),
fits: false,
test: "first tcp port conflict",
},
{
pod: newPod("m1", "TCP/0.0.0.0/8001"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", 8080)),
newPod("m1", "TCP/127.0.0.1/8001")),
fits: false,
test: "same port",
test: "first tcp port conflict due to 0.0.0.0 hostIP",
},
{
pod: newPod("m1", 8000, 8080),
pod: newPod("m1", "TCP/10.0.10.10/8001", "TCP/0.0.0.0/8001"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", 8080)),
newPod("m1", "TCP/127.0.0.1/8001")),
fits: false,
test: "second port",
test: "TCP hostPort conflict due to 0.0.0.0 hostIP",
},
{
pod: newPod("m1", 8000, 8080),
pod: newPod("m1", "TCP/127.0.0.1/8001"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", 8001, 8080)),
newPod("m1", "TCP/0.0.0.0/8001")),
fits: false,
test: "second port",
test: "second tcp port conflict to 0.0.0.0 hostIP",
},
{
pod: newPod("m1", "UDP/127.0.0.1/8001"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "TCP/0.0.0.0/8001")),
fits: true,
test: "second different protocol",
},
{
pod: newPod("m1", "UDP/127.0.0.1/8001"),
nodeInfo: schedulercache.NewNodeInfo(
newPod("m1", "TCP/0.0.0.0/8001", "UDP/0.0.0.0/8001")),
fits: false,
test: "UDP hostPort conflict due to 0.0.0.0 hostIP",
},
}
expectedFailureReasons := []algorithm.PredicateFailureReason{ErrPodNotFitsHostPorts}
Expand All @@ -631,29 +694,28 @@ func TestPodFitsHostPorts(t *testing.T) {

func TestGetUsedPorts(t *testing.T) {
tests := []struct {
pods []*v1.Pod

ports map[int]bool
pods []*v1.Pod
ports map[string]bool
}{
{
[]*v1.Pod{
newPod("m1", 9090),
newPod("m1", "UDP/127.0.0.1/9090"),
},
map[int]bool{9090: true},
map[string]bool{"UDP/127.0.0.1/9090": true},
},
{
[]*v1.Pod{
newPod("m1", 9090),
newPod("m1", 9091),
newPod("m1", "UDP/127.0.0.1/9090"),
newPod("m1", "UDP/127.0.0.1/9091"),
},
map[int]bool{9090: true, 9091: true},
map[string]bool{"UDP/127.0.0.1/9090": true, "UDP/127.0.0.1/9091": true},
},
{
[]*v1.Pod{
newPod("m1", 9090),
newPod("m2", 9091),
newPod("m1", "TCP/0.0.0.0/9090"),
newPod("m2", "UDP/127.0.0.1/9091"),
},
map[int]bool{9090: true, 9091: true},
map[string]bool{"TCP/0.0.0.0/9090": true, "UDP/127.0.0.1/9091": true},
},
}

Expand Down
69 changes: 69 additions & 0 deletions plugin/pkg/scheduler/algorithm/predicates/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,12 @@ limitations under the License.
package predicates

import (
"strings"

"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
schedutil "k8s.io/kubernetes/plugin/pkg/scheduler/util"
)

// FindLabelsInSet gets as many key/value pairs as possible out of a label set.
Expand Down Expand Up @@ -89,3 +92,69 @@ func GetEquivalencePod(pod *v1.Pod) interface{} {
type EquivalencePod struct {
ControllerRef metav1.OwnerReference
}

type hostPortInfo struct {
protocol string
hostIP string
hostPort string
}
Copy link
Contributor

@resouer resouer Oct 9, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The hostPortInfo design looks much cleaner, thanks

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You are far too kind, Now I will move to e2e test


// decode decodes string ("protocol/hostIP/hostPort") to *hostPortInfo object.
func decode(info string) *hostPortInfo {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just nitpick. If it was me, I would define it as something like

func (h *hostPortInfo) Load(info string) error
  • It won't be in the namespace of the whole package.
  • It will return error when it cannot decode the string.
h := hostPostInfo{}
err := h.Load(str)
if err != nil {
	return err
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it is just fine now

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

or at least you call it decodeHostPortInfo. anyway I'll leave it to @k82cn to decide.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

haha I think we can get what we decode from context, not very important to add it to the function name

hostPortInfoSlice := strings.Split(info, "/")

protocol := hostPortInfoSlice[0]
hostIP := hostPortInfoSlice[1]
hostPort := hostPortInfoSlice[2]

return &hostPortInfo{
protocol: protocol,
hostIP: hostIP,
hostPort: hostPort,
}
}

// specialPortConflictCheck detects whether specailHostPort(whose hostIP is 0.0.0.0) is conflict with otherHostPorts.
// return true if we have a conflict.
func specialPortConflictCheck(specialHostPort string, otherHostPorts map[string]bool) bool {
specialHostPortInfo := decode(specialHostPort)

if specialHostPortInfo.hostIP == schedutil.DefaultBindAllHostIP {
// loop through all the otherHostPorts to see if there exists a conflict
for hostPortItem := range otherHostPorts {
hostPortInfo := decode(hostPortItem)

// if there exists one hostPortItem which has the same hostPort and protocol with the specialHostPort, that will cause a conflict
if specialHostPortInfo.hostPort == hostPortInfo.hostPort && specialHostPortInfo.protocol == hostPortInfo.protocol {
return true
}
}

}

return false
}

// portsConflict check whether existingPorts and wantPorts conflict with each other
// return true if we have a conflict
func portsConflict(existingPorts, wantPorts map[string]bool) bool {

for existingPort := range existingPorts {
if specialPortConflictCheck(existingPort, wantPorts) {
return true
}
}

for wantPort := range wantPorts {
if specialPortConflictCheck(wantPort, existingPorts) {
return true
}

// general check hostPort conflict procedure for hostIP is not 0.0.0.0
if existingPorts[wantPort] {
return true
}
}

return false
}