Merge pull request #108460 from Nordix/issue-72236

Prevent host access on VIP addresses in proxy-mode=ipvs

Author: k8s-ci-robot

pkg/proxy/ipvs/ipset.go

@@ -78,6 +78,9 @@ const (
 
 	kubeHealthCheckNodePortSetComment = "Kubernetes health check node port"
 	kubeHealthCheckNodePortSet        = "KUBE-HEALTH-CHECK-NODE-PORT"
+
+	kubeIPVSSetComment = "Addresses on the ipvs interface"
+	kubeIPVSSet        = "KUBE-IPVS-IPS"
 )
 
 // IPSetVersioner can query the current ipset version.

pkg/proxy/ipvs/proxier.go

@@ -84,6 +84,10 @@ const (
 	// kubeLoadBalancerChain is the kubernetes chain for loadbalancer type service
 	kubeLoadBalancerChain utiliptables.Chain = "KUBE-LOAD-BALANCER"
 
+	// kubeIPVSFilterChain filters external access to main netns
+	// https://github.com/kubernetes/kubernetes/issues/72236
+	kubeIPVSFilterChain utiliptables.Chain = "KUBE-IPVS-FILTER"
+
 	// defaultScheduler is the default ipvs scheduler algorithm - round robin.
 	defaultScheduler = "rr"
 
@@ -112,6 +116,7 @@ var iptablesJumpChain = []struct {
 	{utiliptables.TableFilter, utiliptables.ChainInput, kubeNodePortChain, "kubernetes health check rules"},
 	{utiliptables.TableFilter, utiliptables.ChainInput, kubeProxyFirewallChain, "kube-proxy firewall rules"},
 	{utiliptables.TableFilter, utiliptables.ChainForward, kubeProxyFirewallChain, "kube-proxy firewall rules"},
+	{utiliptables.TableFilter, utiliptables.ChainInput, kubeIPVSFilterChain, "kubernetes ipvs access filter"},
 }
 
 var iptablesChains = []struct {
@@ -127,6 +132,7 @@ var iptablesChains = []struct {
 	{utiliptables.TableFilter, kubeNodePortChain},
 	{utiliptables.TableFilter, kubeProxyFirewallChain},
 	{utiliptables.TableFilter, kubeSourceRangesFirewallChain},
+	{utiliptables.TableFilter, kubeIPVSFilterChain},
 }
 
 var iptablesCleanupChains = []struct {
@@ -141,6 +147,7 @@ var iptablesCleanupChains = []struct {
 	{utiliptables.TableFilter, kubeNodePortChain},
 	{utiliptables.TableFilter, kubeProxyFirewallChain},
 	{utiliptables.TableFilter, kubeSourceRangesFirewallChain},
+	{utiliptables.TableFilter, kubeIPVSFilterChain},
 }
 
 // ipsetInfo is all ipset we needed in ipvs proxier
@@ -165,6 +172,7 @@ var ipsetInfo = []struct {
 	{kubeNodePortSetSCTP, utilipset.HashIPPort, kubeNodePortSetSCTPComment},
 	{kubeNodePortLocalSetSCTP, utilipset.HashIPPort, kubeNodePortLocalSetSCTPComment},
 	{kubeHealthCheckNodePortSet, utilipset.BitmapPort, kubeHealthCheckNodePortSetComment},
+	{kubeIPVSSet, utilipset.HashIP, kubeIPVSSetComment},
 }
 
 // ipsetWithIptablesChain is the ipsets list with iptables source chain and the chain jump to
@@ -1549,6 +1557,9 @@ func (proxier *Proxier) syncProxyRules() {
 		}
 	}
 
+	// Set the KUBE-IPVS-IPS set to the "activeBindAddrs"
+	proxier.ipsetList[kubeIPVSSet].activeEntries = sets.StringKeySet(activeBindAddrs)
+
 	// sync ipset entries
 	for _, set := range proxier.ipsetList {
 		set.syncIPSetEntries()
@@ -1792,6 +1803,22 @@ func (proxier *Proxier) writeIptablesRules() {
 		"-j", "ACCEPT",
 	)
 
+	// Add rules to the filter/KUBE-IPVS-FILTER chain to prevent access to ports on the host through VIP addresses.
+	// https://github.com/kubernetes/kubernetes/issues/72236
+	proxier.filterRules.Write(
+		"-A", string(kubeIPVSFilterChain),
+		"-m", "set", "--match-set", proxier.ipsetList[kubeLoadBalancerSet].Name, "dst,dst", "-j", "ACCEPT")
+	proxier.filterRules.Write(
+		"-A", string(kubeIPVSFilterChain),
+		"-m", "set", "--match-set", proxier.ipsetList[kubeClusterIPSet].Name, "dst,dst", "-j", "ACCEPT")
+	proxier.filterRules.Write(
+		"-A", string(kubeIPVSFilterChain),
+		"-m", "set", "--match-set", proxier.ipsetList[kubeExternalIPSet].Name, "dst,dst", "-j", "ACCEPT")
+	proxier.filterRules.Write(
+		"-A", string(kubeIPVSFilterChain),
+		"-m", "conntrack", "--ctstate", "NEW",
+		"-m", "set", "--match-set", proxier.ipsetList[kubeIPVSSet].Name, "dst", "-j", "REJECT")
+
 	// Install the kubernetes-specific postrouting rules. We use a whole chain for
 	// this so that it is easier to flush and change, for example if the mark
 	// value should ever change.

pkg/proxy/ipvs/proxier_test.go

@@ -4772,6 +4772,7 @@ func TestCreateAndLinkKubeChain(t *testing.T) {
 :KUBE-NODE-PORT - [0:0]
 :KUBE-PROXY-FIREWALL - [0:0]
 :KUBE-SOURCE-RANGES-FIREWALL - [0:0]
+:KUBE-IPVS-FILTER - [0:0]
 `
 	assert.Equal(t, expectedNATChains, string(fp.natChains.Bytes()))
 	assert.Equal(t, expectedFilterChains, string(fp.filterChains.Bytes()))

pkg/util/ipset/ipset.go

@@ -169,6 +169,11 @@ func (e *Entry) Validate(set *IPSet) bool {
 		return false
 	}
 	switch e.SetType {
+	case HashIP:
+		//check if IP of Entry is valid.
+		if valid := e.checkIP(set); !valid {
+			return false
+		}
 	case HashIPPort:
 		//check if IP and Protocol of Entry is valid.
 		if valid := e.checkIPandProtocol(set); !valid {
@@ -219,6 +224,9 @@ func (e *Entry) Validate(set *IPSet) bool {
 // String returns the string format for ipset entry.
 func (e *Entry) String() string {
 	switch e.SetType {
+	case HashIP:
+		// Entry{192.168.1.1} -> 192.168.1.1
+		return fmt.Sprintf("%s", e.IP)
 	case HashIPPort:
 		// Entry{192.168.1.1, udp, 53} -> 192.168.1.1,udp:53
 		// Entry{192.168.1.2, tcp, 8080} -> 192.168.1.2,tcp:8080
@@ -247,7 +255,11 @@ func (e *Entry) checkIPandProtocol(set *IPSet) bool {
 	} else if !validateProtocol(e.Protocol) {
 		return false
 	}
+	return e.checkIP(set)
+}
 
+// checkIP checks if IP of Entry is valid.
+func (e *Entry) checkIP(set *IPSet) bool {
 	if netutils.ParseIPSloppy(e.IP) == nil {
 		klog.Errorf("Error parsing entry %v ip address %v for ipset %v", e, e.IP, set)
 		return false
@@ -283,7 +295,7 @@ func (runner *runner) CreateSet(set *IPSet, ignoreExistErr bool) error {
 // otherwise raised when the same set (setname and create parameters are identical) already exists.
 func (runner *runner) createSet(set *IPSet, ignoreExistErr bool) error {
 	args := []string{"create", set.Name, string(set.SetType)}
-	if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet {
+	if set.SetType == HashIPPortIP || set.SetType == HashIPPort || set.SetType == HashIPPortNet || set.SetType == HashIP {
 		args = append(args,
 			"family", set.HashFamily,
 			"hashsize", strconv.Itoa(set.HashSize),

pkg/util/ipset/types.go

@@ -35,6 +35,8 @@ const (
 	// BitmapPort represents the `bitmap:port` type ipset.  The bitmap:port set type uses a memory range, where each bit
 	// represents one TCP/UDP port.  A bitmap:port type of set can store up to 65535 ports.
 	BitmapPort Type = "bitmap:port"
+	// HashIP represents the `hash:ip` type ipset.
+	HashIP Type = "hash:ip"
 )
 
 // DefaultPortRange defines the default bitmap:port valid port range.
@@ -59,4 +61,5 @@ var ValidIPSetTypes = []Type{
 	HashIPPortIP,
 	BitmapPort,
 	HashIPPortNet,
+	HashIP,
 }