Skip to content

Commit

Permalink
Add features: host netns + route source interface (#5)
Browse files Browse the repository at this point in the history 
* add more features to ptp plugin

* add option "host_netns" in configuration to move the host veth
  interface in a specific netns
* add options "route_source_interface_ipv4/ipv6" to change source IP
  used in routes configured by the IPAM plugin
* add option "sysctl", taking as input the same format of configuration
  than the "tuning" plugin to configure sysctl parameters in the
  container netns

* fix: close destNetns fd

* fix: ensure to convert configuration passed to tuning plugin

* result needs to be converted to the right CNI version before passing
  it to another plugin

* fix: remove link-local ipv6 address from the address list

* when using route_source_interface, we expect to have a single IP
  configured on the interface, however in IPv6, there is always a link
  local address configured in addition to the one provided by the IPAM
  plugin
* getIntfIP function was returning an error as we were seeing 2
  addresses instead of 1, now we filter this link-local address to fix
  it

* tests: add tests for new options on ptp plugin

* fix: handle default route IPv4/IPv6 when replacing src IP

* default route in IPv4 and IPv6 is equivalent to a Dst = nil in a route
  filter

* cover new features introduced in ptp in cmdCheck

* we must check that all the new config parameters have been correctly
  set on the host/container netns

* tests: add routes to the test case config

* fix: defer netns closure after checking error

* tests, lint: fix ineffassign errors raised by linter

* lint: remove extra line

* remove mentions to Criteo

* feat: remove sysctl

* the sysctl feature might be unnecessary as we could configure sysctl
  parameters on the host itself, or via the mesos isolator
* we'll repropose that feature again if this is needed
  • Loading branch information
@fdomain
fdomain committed May 29, 2023
1 parent 232366a commit 3ca9d02
Show file tree
Hide file tree
Showing 2 changed files with 359 additions and 33 deletions.
293 changes: 260 additions & 33 deletions plugins/main/ptp/ptp.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,14 @@ func init() {

type NetConf struct {
types.NetConf
IPMasq bool `json:"ipMasq"`
MTU int `json:"mtu"`
IPMasq bool `json:"ipMasq"`
MTU int `json:"mtu"`
HostNetNS string `json:"host_netns"`
RouteSourceInterfaceIPv4 string `json:"route_source_interface_ipv4"`
RouteSourceInterfaceIPv6 string `json:"route_source_interface_ipv6"`
}

func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Result) (*current.Interface, *current.Interface, error) {
func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, routeSrcIntfIPv4 string, routeSrcIntfIPv6 string, pr *current.Result) (*current.Interface, *current.Interface, error) {
// The IPAM result will be something like IP=192.168.3.5/24, GW=192.168.3.1.
// What we want is really a point-to-point link but veth does not support IFF_POINTTOPOINT.
// Next best thing would be to let it ARP but set interface to 192.168.3.5/32 and
Expand Down Expand Up @@ -90,6 +93,20 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
return err
}

var srcIPv4, srcIPv6 net.IP
if routeSrcIntfIPv4 != "" {
srcIPv4, err = getIntfIP(routeSrcIntfIPv4, netlink.FAMILY_V4)
if err != nil {
return err
}
}
if routeSrcIntfIPv6 != "" {
srcIPv6, err = getIntfIP(routeSrcIntfIPv6, netlink.FAMILY_V6)
if err != nil {
return err
}
}

for _, ipc := range pr.IPs {
// Delete the route that was automatically added
route := netlink.Route{
Expand Down Expand Up @@ -137,6 +154,17 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
}
}

if srcIPv4 != nil {
if err := replaceRouteSrcIP(contVeth.Index, srcIPv4, pr.Routes); err != nil {
return err
}
}

if srcIPv6 != nil {
if err := replaceRouteSrcIP(contVeth.Index, srcIPv6, pr.Routes); err != nil {
return err
}
}
return nil
})
if err != nil {
Expand All @@ -145,39 +173,135 @@ func setupContainerVeth(netns ns.NetNS, ifName string, mtu int, pr *current.Resu
return hostInterface, containerInterface, nil
}

func setupHostVeth(vethName string, result *current.Result) error {
// hostVeth moved namespaces and may have a new ifindex
veth, err := netlink.LinkByName(vethName)
func removeLinkLocalAddresses(addrList []netlink.Addr) []netlink.Addr {
var res []netlink.Addr
for _, addr := range addrList {
// we removed link-local addresses
if addr.Scope != int(netlink.SCOPE_UNIVERSE) {
continue
}
res = append(res, addr)
}
return res
}

// getIntfIP returns the primary IP configured on the ifName interface for the given family.
func getIntfIP(ifName string, family int) (net.IP, error) {
sourceIntf, err := netlink.LinkByName(ifName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", vethName, err)
return nil, fmt.Errorf("failed to lookup %q: %v", ifName, err)
}

for _, ipc := range result.IPs {
maskLen := 128
if ipc.Address.IP.To4() != nil {
maskLen = 32
}
addrList, err := netlink.AddrList(sourceIntf, family)
addrList = removeLinkLocalAddresses(addrList)
if err != nil {
return nil, fmt.Errorf("cannot obtain list of IP addresses for %s: %v", sourceIntf.Attrs().Name, err)
}
if len(addrList) != 1 {
return nil, fmt.Errorf("no address or more than one address configured on interface %s", sourceIntf.Attrs().Name)
}

ipn := &net.IPNet{
IP: ipc.Gateway,
Mask: net.CIDRMask(maskLen, maskLen),
return addrList[0].IP, nil
}

// replaceRouteSrcIP replaces the source IP used for the routes attached to a link.
func replaceRouteSrcIP(linkIndex int, srcIP net.IP, routes []*types.Route) error {
family := netlink.FAMILY_V4
isV4 := srcIP.To4() != nil
if !isV4 {
family = netlink.FAMILY_V6
}
for _, r := range routes {
if (r.Dst.IP.To4() != nil) != isV4 {
continue
}
filter := &netlink.Route{
LinkIndex: linkIndex,
Dst: &r.Dst,
}
addr := &netlink.Addr{IPNet: ipn, Label: ""}
if err = netlink.AddrAdd(veth, addr); err != nil {
return fmt.Errorf("failed to add IP addr (%#v) to veth: %v", ipn, err)
if r.Dst.String() == "0.0.0.0/0" || r.Dst.String() == "::/0" {
filter.Dst = nil
}
routeList, err := netlink.RouteListFiltered(family, filter, netlink.RT_FILTER_DST)
if err != nil {
return fmt.Errorf("cannot obtain list of routes for link index %d: %v", linkIndex, err)
}
if len(routeList) > 1 {
return fmt.Errorf("%d routes found for %s", len(routeList), r.Dst.String())
} else if len(routeList) == 0 {
return fmt.Errorf("no route found for %s", r.Dst.String())
}
newRoute := routeList[0]
newRoute.Src = srcIP
if err := netlink.RouteReplace(&newRoute); err != nil {
return fmt.Errorf("failed to replace route: %v", err)
}
}
return nil
}

ipn = &net.IPNet{
IP: ipc.Address.IP,
Mask: net.CIDRMask(maskLen, maskLen),
// setupHostVeth configure the veth interface on the host side, optionally moving it
// to a different netns than the one used to invoke the script.
func setupHostVeth(netns string, vethName string, result *current.Result) error {
// vethName moved to another namespace and may have a new ifindex
veth, err := netlink.LinkByName(vethName)
if err != nil {
return fmt.Errorf("failed to lookup %q: %v", vethName, err)
}

var destNetns ns.NetNS
// move hostVeth in the target NetNS
if netns != "" {
destNetns, err = ns.GetNS(fmt.Sprintf("/var/run/netns/%s", netns))
if err != nil {
return fmt.Errorf("failed to get netns %s", netns)
}
// dst happens to be the same as IP/net of host veth
if err = ip.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to add route on host: %v", err)
defer destNetns.Close()
if err := netlink.LinkSetNsFd(veth, int(destNetns.Fd())); err != nil {
return fmt.Errorf("failed to move host veth to target netns %s: %v", netns, err)
}
err = destNetns.Do(func(hostNS ns.NetNS) error {
return netlink.LinkSetUp(veth)
})
if err != nil {
return fmt.Errorf("failed to set up veth in target netns %s: %v", netns, err)
}
} else {
destNetns, err = ns.GetCurrentNS()
if err != nil {
return fmt.Errorf("failed to get current netns")
}
defer destNetns.Close()
}

return nil
return destNetns.Do(func(hostNS ns.NetNS) error {
for _, ipc := range result.IPs {
maskLen := 128
if ipc.Address.IP.To4() != nil {
maskLen = 32
}

ipn := &net.IPNet{
IP: ipc.Gateway,
Mask: net.CIDRMask(maskLen, maskLen),
}
addr := &netlink.Addr{IPNet: ipn, Label: ""}
if err = netlink.AddrAdd(veth, addr); err != nil {
return fmt.Errorf("failed to add IP addr (%#v) to veth: %v", ipn, err)
}

ipn = &net.IPNet{
IP: ipc.Address.IP,
Mask: net.CIDRMask(maskLen, maskLen),
}
// dst happens to be the same as IP/net of host veth
if err = ip.AddHostRoute(ipn, nil, veth); err != nil && !os.IsExist(err) {
return fmt.Errorf("failed to add route on host: %v", err)
}
}

return nil
})
}

func cmdAdd(args *skel.CmdArgs) error {
Expand Down Expand Up @@ -213,18 +337,25 @@ func cmdAdd(args *skel.CmdArgs) error {
return fmt.Errorf("Could not enable IP forwarding: %v", err)
}

netns, err := ns.GetNS(args.Netns)
contNetns, err := ns.GetNS(args.Netns)
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
}
defer netns.Close()

hostInterface, _, err := setupContainerVeth(netns, args.IfName, conf.MTU, result)
return fmt.Errorf("failed to open container netns %q: %v", args.Netns, err)
}
defer contNetns.Close()

hostInterface, _, err := setupContainerVeth(
contNetns,
args.IfName,
conf.MTU,
conf.RouteSourceInterfaceIPv4,
conf.RouteSourceInterfaceIPv6,
result,
)
if err != nil {
return err
}

if err = setupHostVeth(hostInterface.Name, result); err != nil {
if err = setupHostVeth(conf.HostNetNS, hostInterface.Name, result); err != nil {
return err
}

Expand Down Expand Up @@ -336,7 +467,7 @@ func cmdCheck(args *skel.CmdArgs) error {
return err
}

var contMap current.Interface
var contMap, hostMap current.Interface
// Find interfaces for name whe know, that of host-device inside container
for _, intf := range result.Interfaces {
if args.IfName == intf.Name {
Expand All @@ -345,6 +476,8 @@ func cmdCheck(args *skel.CmdArgs) error {
continue
}
}
// we assume the other interface corresponds to the host's interface
hostMap = *intf
}

// The namespace must be the same as what was configured
Expand All @@ -353,6 +486,23 @@ func cmdCheck(args *skel.CmdArgs) error {
contMap.Sandbox, args.Netns)
}

if conf.HostNetNS != "" {
hostNetns, err := ns.GetNS(fmt.Sprintf("/var/run/netns/%s", conf.HostNetNS))
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", conf.HostNetNS, err)
}
defer hostNetns.Close()
if err := hostNetns.Do(func(_ ns.NetNS) error {
err := validateCniHostInterface(hostMap)
if err != nil {
return err
}
return nil
}); err != nil {
return err
}
}

//
// Check prevResults for ips, routes and dns against values found in the container
if err := netns.Do(func(_ ns.NetNS) error {
Expand All @@ -371,6 +521,20 @@ func cmdCheck(args *skel.CmdArgs) error {
if err != nil {
return err
}

if conf.RouteSourceInterfaceIPv4 != "" {
err = validateSourceIPRoute(conf.RouteSourceInterfaceIPv4, netlink.FAMILY_V4, result.Routes)
if err != nil {
return err
}
}

if conf.RouteSourceInterfaceIPv6 != "" {
err = validateSourceIPRoute(conf.RouteSourceInterfaceIPv6, netlink.FAMILY_V6, result.Routes)
if err != nil {
return err
}
}
return nil
}); err != nil {
return err
Expand Down Expand Up @@ -407,3 +571,66 @@ func validateCniContainerInterface(intf current.Interface) error {

return nil
}

func validateCniHostInterface(intf current.Interface) error {
link, err := netlink.LinkByName(intf.Name)
if err != nil {
return fmt.Errorf("ptp: Host Interface name in prevResult: %s not found", intf.Name)
}
_, isVeth := link.(*netlink.Veth)
if !isVeth {
return fmt.Errorf("Error: Host interface %s not of type veth/p2p", link.Attrs().Name)
}
return nil
}

func validateSourceIPRoute(ifaceSrcIP string, family int, resultRoutes []*types.Route) error {
srcIP, err := getIntfIP(ifaceSrcIP, family)
if err != nil {
return err
}

var filteredRoutes []*types.Route

for _, route := range resultRoutes {
if family == netlink.FAMILY_V4 && route.Dst.IP.To4() != nil || family == netlink.FAMILY_V6 && route.Dst.IP.To16() != nil {
filteredRoutes = append(filteredRoutes, route)
}
}

// Ensure that each static route has the correct source IP
for _, route := range filteredRoutes {
find := &netlink.Route{Dst: &route.Dst, Gw: route.GW, Src: srcIP}
routeFilter := netlink.RT_FILTER_DST | netlink.RT_FILTER_SRC
if route.GW != nil {
routeFilter |= netlink.RT_FILTER_GW
}

switch {
case route.Dst.IP.To4() != nil:
family = netlink.FAMILY_V4
// Default route needs Dst set to nil
if route.Dst.String() == "0.0.0.0/0" {
find = &netlink.Route{Dst: nil, Gw: route.GW, Src: srcIP}
}
case len(route.Dst.IP) == net.IPv6len:
family = netlink.FAMILY_V6
// Default route needs Dst set to nil
if route.Dst.String() == "::/0" {
find = &netlink.Route{Dst: nil, Gw: route.GW, Src: srcIP}
}
default:
return fmt.Errorf("Invalid static route found %v", route)
}

wasFound, err := netlink.RouteListFiltered(family, find, routeFilter)
if err != nil {
return fmt.Errorf("Expected Route %v not route table lookup error %v", route, err)
}
if wasFound == nil {
return fmt.Errorf("Expected Route %v not found in routing table", route)
}
}

return nil
}

0 comments on commit 3ca9d02

Please sign in to comment.