Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add NodeCIDR for detect-local-mode #88935

Merged
merged 1 commit into from
Mar 9, 2020
+281 −12
Merged

Add NodeCIDR for detect-local-mode #88935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions cmd/kube-proxy/app/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -92,6 +95,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -101,6 +107,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -110,6 +119,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -119,6 +131,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -128,6 +143,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -137,6 +155,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -146,6 +167,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -155,6 +179,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -164,6 +191,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand All @@ -173,6 +203,9 @@ go_library(
"//pkg/util/node:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/types:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library",
"//staging/src/k8s.io/client-go/tools/cache:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/component-base/metrics:go_default_library",
"//vendor/k8s.io/utils/net:go_default_library",
],
Expand Down Expand Up @@ -211,66 +244,77 @@ go_test(
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:darwin": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:dragonfly": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:freebsd": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:ios": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:linux": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:nacl": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:netbsd": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:openbsd": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:plan9": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"@io_bazel_rules_go//go/platform:solaris": [
"//pkg/proxy/ipvs:go_default_library",
"//pkg/proxy/util/iptables:go_default_library",
"//pkg/util/iptables:go_default_library",
"//pkg/util/iptables/testing:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
],
"//conditions:default": [],
}),
Expand Down
105 changes: 98 additions & 7 deletions cmd/kube-proxy/app/server_others.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,29 @@ limitations under the License.
package app

import (
"context"
"errors"
"fmt"
"net"
"strings"
"time"

"k8s.io/apimachinery/pkg/watch"

"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/tools/cache"

"k8s.io/apimachinery/pkg/fields"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
utilnet "k8s.io/apimachinery/pkg/util/net"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
utilfeature "k8s.io/apiserver/pkg/util/feature"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/tools/record"
toolswatch "k8s.io/client-go/tools/watch"
"k8s.io/component-base/metrics"
"k8s.io/kubernetes/pkg/features"
"k8s.io/kubernetes/pkg/proxy"
Expand All @@ -55,6 +67,10 @@ import (
"k8s.io/klog"
)

// timeoutForNodePodCIDR is the time to wait for allocators to assign a PodCIDR to the
// node after it is registered.
var timeoutForNodePodCIDR = 5 * time.Minute
satyasm marked this conversation as resolved.
Show resolved Hide resolved

// NewProxyServer returns a new ProxyServer.
func NewProxyServer(o *Options) (*ProxyServer, error) {
return newProxyServer(o.config, o.CleanupAndExit, o.master)
Expand Down Expand Up @@ -145,6 +161,16 @@ func newProxyServer(
return nil, fmt.Errorf("cannot determine detect-local-mode: %v", err)
}

var nodeInfo *v1.Node
if detectLocalMode == proxyconfigapi.LocalModeNodeCIDR {
klog.Infof("Watching for node %s, awaiting podCIDR allocation", hostname)
nodeInfo, err = waitForPodCIDR(client, hostname)
if err != nil {
return nil, err
}
klog.Infof("NodeInfo PodCIDR: %v, PodCIDRs: %v", nodeInfo.Spec.PodCIDR, nodeInfo.Spec.PodCIDRs)
}

nodeIP := net.ParseIP(config.BindAddress)
if nodeIP.IsUnspecified() {
nodeIP = utilnode.GetNodeIP(client, hostname)
Expand Down Expand Up @@ -179,7 +205,7 @@ func newProxyServer(

// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt)
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
Expand All @@ -202,7 +228,7 @@ func newProxyServer(
)
} else { // Create a single-stack proxier.
var localDetector proxyutiliptables.LocalTrafficDetector
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface)
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
Expand Down Expand Up @@ -249,7 +275,7 @@ func newProxyServer(

// Always ordered to match []ipt
var localDetectors [2]proxyutiliptables.LocalTrafficDetector
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt)
localDetectors, err = getDualStackLocalDetectorTuple(detectLocalMode, config, ipt, nodeInfo)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
Expand Down Expand Up @@ -279,7 +305,7 @@ func newProxyServer(
)
} else {
var localDetector proxyutiliptables.LocalTrafficDetector
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface)
localDetector, err = getLocalDetector(detectLocalMode, config, iptInterface, nodeInfo)
if err != nil {
return nil, fmt.Errorf("unable to create proxier: %v", err)
}
Expand Down Expand Up @@ -355,10 +381,44 @@ func newProxyServer(
}, nil
}

func waitForPodCIDR(client clientset.Interface, nodeName string) (*v1.Node, error) {
// since allocators can assign the podCIDR after the node registers, we do a watch here to wait
// for podCIDR to be assigned, instead of assuming that the Get() on startup will have it.
ctx, cancelFunc := context.WithTimeout(context.TODO(), timeoutForNodePodCIDR)
defer cancelFunc()

fieldSelector := fields.OneTermEqualSelector("metadata.name", nodeName).String()
lw := &cache.ListWatch{
ListFunc: func(options metav1.ListOptions) (object runtime.Object, e error) {
options.FieldSelector = fieldSelector
return client.CoreV1().Nodes().List(ctx, options)
},
WatchFunc: func(options metav1.ListOptions) (i watch.Interface, e error) {
options.FieldSelector = fieldSelector
return client.CoreV1().Nodes().Watch(ctx, options)
},
}
condition := func(event watch.Event) (bool, error) {
if n, ok := event.Object.(*v1.Node); ok {
return n.Spec.PodCIDR != "" && len(n.Spec.PodCIDRs) > 0, nil
}
return false, fmt.Errorf("event object not of type Node")
}

evt, err := toolswatch.UntilWithSync(ctx, lw, &v1.Node{}, nil, condition)
if err != nil {
return nil, fmt.Errorf("timeout waiting for PodCIDR allocation to configure detect-local-mode %v: %v", proxyconfigapi.LocalModeNodeCIDR, err)
}
if n, ok := evt.Object.(*v1.Node); ok {
return n, nil
}
return nil, fmt.Errorf("event object not of type node")
}

func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxyconfigapi.LocalMode, error) {
mode := config.DetectLocalMode
switch mode {
satyasm marked this conversation as resolved.
Show resolved Hide resolved
case proxyconfigapi.LocalModeClusterCIDR:
case proxyconfigapi.LocalModeClusterCIDR, proxyconfigapi.LocalModeNodeCIDR:
return mode, nil
default:
if strings.TrimSpace(mode.String()) != "" {
Expand All @@ -369,20 +429,26 @@ func getDetectLocalMode(config *proxyconfigapi.KubeProxyConfiguration) (proxycon
}
}

func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface) (proxyutiliptables.LocalTrafficDetector, error) {
func getLocalDetector(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt utiliptables.Interface, nodeInfo *v1.Node) (proxyutiliptables.LocalTrafficDetector, error) {
switch mode {
case proxyconfigapi.LocalModeClusterCIDR:
if len(strings.TrimSpace(config.ClusterCIDR)) == 0 {
klog.Warning("detect-local-mode set to ClusterCIDR, but no cluster CIDR defined")
break
}
return proxyutiliptables.NewDetectLocalByCIDR(config.ClusterCIDR, ipt)
case proxyconfigapi.LocalModeNodeCIDR:
if len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 {
klog.Warning("detect-local-mode set to NodeCIDR, but no PodCIDR defined at node")
break
}
return proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt)
}
klog.V(0).Info("detect-local-mode: ", string(mode), " , defaulting to no-op detect-local")
return proxyutiliptables.NewNoOpLocalDetector(), nil
}

func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface) ([2]proxyutiliptables.LocalTrafficDetector, error) {
func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxyconfigapi.KubeProxyConfiguration, ipt [2]utiliptables.Interface, nodeInfo *v1.Node) ([2]proxyutiliptables.LocalTrafficDetector, error) {
var err error
localDetectors := [2]proxyutiliptables.LocalTrafficDetector{proxyutiliptables.NewNoOpLocalDetector(), proxyutiliptables.NewNoOpLocalDetector()}
switch mode {
Expand All @@ -409,6 +475,31 @@ func getDualStackLocalDetectorTuple(mode proxyconfigapi.LocalMode, config *proxy
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(clusterCIDRs[1], ipt[1])
}
return localDetectors, err
case proxyconfigapi.LocalModeNodeCIDR:
if nodeInfo == nil || len(strings.TrimSpace(nodeInfo.Spec.PodCIDR)) == 0 {
klog.Warning("No node info available to configure detect-local-mode NodeCIDR")
break
}
// localDetectors, like ipt, need to be of the order [IPv4, IPv6], but PodCIDRs is setup so that PodCIDRs[0] == PodCIDR.
// so have to handle the case where PodCIDR can be IPv6 and set that to localDetectors[1]
if utilsnet.IsIPv6CIDRString(nodeInfo.Spec.PodCIDR) {
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[1])
if err != nil {
return localDetectors, err
}
if len(nodeInfo.Spec.PodCIDRs) > 1 {
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[0])
}
} else {
localDetectors[0], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDR, ipt[0])
if err != nil {
return localDetectors, err
}
if len(nodeInfo.Spec.PodCIDRs) > 1 {
localDetectors[1], err = proxyutiliptables.NewDetectLocalByCIDR(nodeInfo.Spec.PodCIDRs[1], ipt[1])
}
}
return localDetectors, err
default:
klog.Warningf("unknown detect-local-mode: %v", mode)
}
Expand Down
Loading