podnetwork: Support CNI plugins like PTP and GKE

CNI plugins like PTP and GKE remove a route that is automatically added by kernel for eth0, and then add another route for the same destination. This patch changes the code to manipulates routes to support such CNI plugins. Fixes #1909 Signed-off-by: Yohei Ueda <yohei@jp.ibm.com>
confidential-containers · Aug 3, 2024 · e18ef16 · e18ef16
1 parent 65bcead
commit e18ef16
Show file tree

Hide file tree

Showing 4 changed files with 79 additions and 37 deletions.
diff --git a/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go b/src/cloud-api-adaptor/pkg/podnetwork/podnetwork_test.go
@@ -99,10 +99,13 @@ func TestWorkerNode(t *testing.T) {
 			require.Equal(t, expected.workerNodeIP, config.WorkerNodeIP.String(), "hostInterface=%q", hostInterface)
 			require.Equal(t, mockTunnelType, config.TunnelType, "hostInterface=%q", hostInterface)
 
-			require.Equal(t, len(config.Routes), 1, "hostInterface=%q", hostInterface)
+			require.Equal(t, len(config.Routes), 2, "hostInterface=%q", hostInterface)
 			require.Equal(t, config.Routes[0].Dst.String(), "0.0.0.0/0", "hostInterface=%q", hostInterface)
 			require.Equal(t, config.Routes[0].GW.String(), "172.16.0.1", "hostInterface=%q", hostInterface)
 			require.Equal(t, config.Routes[0].Dev, "eth0", "hostInterface=%q", hostInterface)
+			require.Equal(t, config.Routes[1].Dst.String(), "172.16.0.0/24", "hostInterface=%q", hostInterface)
+			require.Equal(t, config.Routes[1].GW.IsValid(), false, "hostInterface=%q", hostInterface)
+			require.Equal(t, config.Routes[1].Dev, "eth0", "hostInterface=%q", hostInterface)
 
 			err = workerNode.Teardown(workerPodNS.Path(), config)
 			require.Nil(t, err, "hostInterface=%q", hostInterface)
@@ -128,9 +131,6 @@ func TestPodNode(t *testing.T) {
 	tuntest.AddrAdd(t, podNodeNS, "ens1", "192.168.1.3/24")
 	tuntest.RouteAdd(t, podNodeNS, "", "192.168.0.1", "ens0")
 
-	podNS := tuntest.NewNamedNS(t, "test-pod")
-	defer tuntest.DeleteNamedNS(t, podNS)
-
 	for hostInterface, expected := range map[string]struct {
 		podNodeIP    string
 		workerNodeIP string
@@ -149,35 +149,48 @@ func TestPodNode(t *testing.T) {
 		},
 	} {
 
-		err := podNodeNS.Run(func() error {
-
-			config := &tunneler.Config{
-				PodIP: netip.MustParsePrefix("172.16.0.2/24"),
-				Routes: []*tunneler.Route{
-					{
-						GW:  netip.MustParseAddr("172.16.0.1"),
-						Dev: "eth0",
+		podNS := tuntest.NewNamedNS(t, "test-pod")
+		func() {
+			defer tuntest.DeleteNamedNS(t, podNS)
+
+			tuntest.BridgeAdd(t, podNS, "eth0")
+			tuntest.AddrAdd(t, podNS, "eth0", "172.16.0.2/24")
+
+			err := podNodeNS.Run(func() error {
+
+				config := &tunneler.Config{
+					PodIP: netip.MustParsePrefix("172.16.0.2/24"),
+					Routes: []*tunneler.Route{
+						{
+							Dst: netip.MustParsePrefix("0.0.0.0/0"),
+							GW:  netip.MustParseAddr("172.16.0.1"),
+							Dev: "eth0",
+						},
+						{
+							Dst: netip.MustParsePrefix("172.16.0.0/24"),
+							Dev: "eth0",
+						},
 					},
-				},
-				InterfaceName: "eth0",
-				MTU:           1500,
-				WorkerNodeIP:  netip.MustParsePrefix(expected.workerNodeIP),
-				TunnelType:    mockTunnelType,
-				Dedicated:     hostInterface == "ens1",
-			}
-
-			podNode := NewPodNode(podNS.Path(), hostInterface, config)
-			require.NotNil(t, podNode, "hostInterface=%q", hostInterface)
-
-			err := podNode.Setup()
-			require.Nil(t, err, "hostInterface=%q", hostInterface)
+					InterfaceName: "eth0",
+					MTU:           1500,
+					WorkerNodeIP:  netip.MustParsePrefix(expected.workerNodeIP),
+					TunnelType:    mockTunnelType,
+					Dedicated:     hostInterface == "ens1",
+				}
 
-			err = podNode.Teardown()
-			require.Nil(t, err, "hostInterface=%q", hostInterface)
+				podNode := NewPodNode(podNS.Path(), hostInterface, config)
+				require.NotNil(t, podNode, "hostInterface=%q", hostInterface)
 
-			return nil
-		})
-		require.Nil(t, err, "hostInterface=%q", hostInterface)
+				err := podNode.Setup()
+				require.Nil(t, err, "hostInterface=%q", hostInterface)
+
+				err = podNode.Teardown()
+				require.Nil(t, err, "hostInterface=%q", hostInterface)
+
+				return nil
+			})
+			require.Nil(t, err, "hostInterface=%q", hostInterface)
+		}()
 	}
 }
 

diff --git a/src/cloud-api-adaptor/pkg/podnetwork/podnode.go b/src/cloud-api-adaptor/pkg/podnetwork/podnode.go
@@ -95,6 +95,22 @@ func (n *podNode) Setup() error {
 		return fmt.Errorf("failed to set up tunnel %q: %w", n.config.TunnelType, err)
 	}
 
+	if !n.config.PodIP.IsSingleIP() {
+		// Delete the nRoute that was automatically added by kernel for eth0
+		// CNI plugins like PTP and GKE need this trick, otherwise adding a route will fail in a later step.
+		// The deleted route will be restored again in the cases of usual CNI plugins such as Flannel and Calico.
+		// https://github.com/containernetworking/plugins/blob/acf8ddc8e1128e6f68a34f7fe91122afeb1fa93d/plugins/main/ptp/ptp.go#L58-L61
+
+		nRoute := netops.Route{
+			Destination: n.config.PodIP.Masked(),
+			Device:      n.config.InterfaceName,
+		}
+		if err := podNS.RouteDel(&nRoute); err != nil {
+			return fmt.Errorf("failed to remove route %s dev %s: %v", nRoute.Destination, nRoute.Device, err)
+		}
+		logger.Printf("removed route %s dev %s", nRoute.Destination, nRoute.Device)
+	}
+
 	// We need to process routes without gateway address first. Processing routes with a gateway causes an error if the gateway is not reachable.
 	// Calico sets up routes with this pattern.
 	// https://github.com/projectcalico/cni-plugin/blob/7495c0279c34faac315b82c1838bca638e23dbbe/pkg/dataplane/linux/dataplane_linux.go#L158-L167
@@ -110,7 +126,14 @@ func (n *podNode) Setup() error {
 	routes := append(first, second...)
 
 	for _, route := range routes {
-		if err := podNS.RouteAdd(&netops.Route{Destination: route.Dst, Gateway: route.GW, Device: route.Dev}); err != nil {
+		nRoute := netops.Route{
+			Destination: route.Dst,
+			Gateway:     route.GW,
+			Device:      route.Dev,
+			Protocol:    route.Protocol,
+			Scope:       route.Scope,
+		}
+		if err := podNS.RouteAdd(&nRoute); err != nil {
 			return fmt.Errorf("failed to add a route to %s via %s on pod network namespace %s: %w", route.Dst, route.GW, podNS.Path(), err)
 		}
 	}

diff --git a/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go b/src/cloud-api-adaptor/pkg/podnetwork/tunneler/tunneler.go
@@ -6,6 +6,8 @@ package tunneler
 import (
 	"fmt"
 	"net/netip"
+
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/util/netops"
 )
 
 type Tunneler interface {
@@ -28,9 +30,11 @@ type Config struct {
 }
 
 type Route struct {
-	Dst netip.Prefix
-	GW  netip.Addr
-	Dev string
+	Dst      netip.Prefix
+	GW       netip.Addr
+	Dev      string
+	Protocol netops.RouteProtocol
+	Scope    netops.RouteScope
 }
 
 type driver struct {

diff --git a/src/cloud-api-adaptor/pkg/podnetwork/workernode.go b/src/cloud-api-adaptor/pkg/podnetwork/workernode.go
@@ -164,9 +164,11 @@ func (n *workerNode) Inspect(nsPath string) (*tunneler.Config, error) {
 
 	for _, route := range routes {
 		r := &tunneler.Route{
-			Dst: route.Destination,
-			Dev: route.Device,
-			GW:  route.Gateway,
+			Dst:      route.Destination,
+			Dev:      route.Device,
+			GW:       route.Gateway,
+			Protocol: route.Protocol,
+			Scope:    route.Scope,
 		}
 		config.Routes = append(config.Routes, r)
 	}