Merge pull request #581 from danwinship/nftables-cni

SDN-4114: Do per-pod MCS/metadata blocking with nftables rather than iptables

Author: openshift-merge-bot[bot]

go.mod

@@ -30,6 +30,7 @@ require (
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kubernetes v1.28.3 // actual branch sdn-4.15-kubernetes-1.28.3 in openshift/kubernetes
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
+	sigs.k8s.io/knftables v0.0.16
 	sigs.k8s.io/yaml v1.3.0
 )
 

go.sum

@@ -1067,6 +1067,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 h1:trsWhjU5jZrx6U
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2/go.mod h1:+qG7ISXqCDVVcyO8hLn12AKVYYUjM7ftlqsqmrhMZE0=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
+sigs.k8s.io/knftables v0.0.16 h1:ZpTfNsjnidgoXdxxzcZLdSctqkpSO3QB3jo3zQ4PXqM=
+sigs.k8s.io/knftables v0.0.16/go.mod h1:f/5ZLKYEUPUhVjUCg6l80ACdL7CIIyeL0DxfgojGRTk=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs=

pkg/cmd/openshift-sdn-cni/openshift-sdn.go

@@ -2,13 +2,13 @@ package openshift_sdn_cni
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"os"
-	"os/exec"
 	"strings"
 
 	configv1 "github.com/openshift/api/config/v1"
@@ -21,8 +21,8 @@ import (
 	"github.com/containernetworking/plugins/pkg/ip"
 	"github.com/containernetworking/plugins/pkg/ipam"
 	"github.com/containernetworking/plugins/pkg/ns"
-
 	"github.com/vishvananda/netlink"
+	"sigs.k8s.io/knftables"
 )
 
 type cniPlugin struct {
@@ -116,24 +116,73 @@ func (p *cniPlugin) testCmdAdd(args *skel.CmdArgs) (types.Result, error) {
 	return convertToRequestedVersion(args.StdinData, result)
 }
 
-func generateIPTablesCommands(platformType string) [][]string {
+func doNFTablesRules(platformType string) error {
+	nft, err := knftables.New(knftables.IPv4Family, "openshift-block-output")
+	if err != nil {
+		return err
+	}
+
+	tx := nft.NewTransaction()
+	tx.Add(&knftables.Table{})
+
+	tx.Add(&knftables.Chain{
+		Name: "block",
+	})
+
+	// Block MCS
+	tx.Add(&knftables.Rule{
+		Chain: "block",
+		Rule: knftables.Concat(
+			"tcp dport { 22623, 22624 } tcp flags syn",
+			"reject",
+		),
+	})
+
+	// Block cloud provider metadata IP except DNS
 	metadataServiceIP := "169.254.169.254"
 	if platformType == string(configv1.AlibabaCloudPlatformType) {
 		metadataServiceIP = "100.100.100.200"
 	}
-	return [][]string{
-		// Block MCS
-		{"-A", "OUTPUT", "-p", "tcp", "-m", "tcp", "--dport", "22623", "--syn", "-j", "REJECT"},
-		{"-A", "OUTPUT", "-p", "tcp", "-m", "tcp", "--dport", "22624", "--syn", "-j", "REJECT"},
-		{"-A", "FORWARD", "-p", "tcp", "-m", "tcp", "--dport", "22623", "--syn", "-j", "REJECT"},
-		{"-A", "FORWARD", "-p", "tcp", "-m", "tcp", "--dport", "22624", "--syn", "-j", "REJECT"},
-
-		// Block cloud provider metadata IP except DNS
-		{"-A", "OUTPUT", "-p", "tcp", "-m", "tcp", "-d", metadataServiceIP, "!", "--dport", "53", "-j", "REJECT"},
-		{"-A", "OUTPUT", "-p", "udp", "-m", "udp", "-d", metadataServiceIP, "!", "--dport", "53", "-j", "REJECT"},
-		{"-A", "FORWARD", "-p", "tcp", "-m", "tcp", "-d", metadataServiceIP, "!", "--dport", "53", "-j", "REJECT"},
-		{"-A", "FORWARD", "-p", "udp", "-m", "udp", "-d", metadataServiceIP, "!", "--dport", "53", "-j", "REJECT"},
-	}
+	tx.Add(&knftables.Rule{
+		Chain: "block",
+		Rule: knftables.Concat(
+			"ip daddr", metadataServiceIP,
+			"udp dport != 53",
+			"reject",
+		),
+	})
+	tx.Add(&knftables.Rule{
+		Chain: "block",
+		Rule: knftables.Concat(
+			"ip daddr", metadataServiceIP,
+			"tcp dport != 53",
+			"reject",
+		),
+	})
+
+	tx.Add(&knftables.Chain{
+		Name:     "output",
+		Type:     knftables.PtrTo(knftables.FilterType),
+		Hook:     knftables.PtrTo(knftables.OutputHook),
+		Priority: knftables.PtrTo(knftables.FilterPriority),
+	})
+	tx.Add(&knftables.Rule{
+		Chain: "output",
+		Rule:  "goto block",
+	})
+
+	tx.Add(&knftables.Chain{
+		Name:     "forward",
+		Type:     knftables.PtrTo(knftables.FilterType),
+		Hook:     knftables.PtrTo(knftables.ForwardHook),
+		Priority: knftables.PtrTo(knftables.FilterPriority),
+	})
+	tx.Add(&knftables.Rule{
+		Chain: "forward",
+		Rule:  "goto block",
+	})
+
+	return nft.Run(context.Background(), tx)
 }
 
 func (p *cniPlugin) CmdAdd(args *skel.CmdArgs) error {
@@ -253,11 +302,8 @@ func (p *cniPlugin) CmdAdd(args *skel.CmdArgs) error {
 		}
 
 		// Block access to certain things
-		for _, args := range generateIPTablesCommands(config.PlatformType) {
-			out, err := exec.Command("iptables", append([]string{"-w"}, args...)...).CombinedOutput()
-			if err != nil {
-				return fmt.Errorf("could not set up pod iptables rules: %s", string(out))
-			}
+		if err = doNFTablesRules(config.PlatformType); err != nil {
+			return fmt.Errorf("could not set up pod nftables rules: %v", err)
 		}
 
 		return nil

vendor/modules.txt

@@ -1486,6 +1486,9 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client/proto/client
 ## explicit; go 1.18
 sigs.k8s.io/json
 sigs.k8s.io/json/internal/golang/encoding/json
+# sigs.k8s.io/knftables v0.0.16
+## explicit; go 1.20
+sigs.k8s.io/knftables
 # sigs.k8s.io/structured-merge-diff/v4 v4.2.3
 ## explicit; go 1.13
 sigs.k8s.io/structured-merge-diff/v4/fieldpath