Skip to content

Commit

Permalink
add support for custom routes
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Aug 22, 2021
1 parent 8339e02 commit 96b0c11
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 65 deletions.
16 changes: 8 additions & 8 deletions cmd/cni/cni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ import (
"runtime"
"strings"

"github.com/kubeovn/kube-ovn/pkg/util"

"github.com/containernetworking/cni/pkg/skel"
"github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/types/current"
"github.com/containernetworking/cni/pkg/version"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/request"
"github.com/kubeovn/kube-ovn/pkg/util"
)

func init() {
Expand All @@ -30,8 +30,6 @@ func main() {
}

func cmdAdd(args *skel.CmdArgs) error {
var err error

netConf, cniVersion, err := loadNetConf(args.StdinData)
if err != nil {
return err
Expand All @@ -49,7 +47,6 @@ func cmdAdd(args *skel.CmdArgs) error {
}

client := request.NewCniServerClient(netConf.ServerSocket)

response, err := client.Add(request.CniRequest{
CniType: netConf.Type,
PodName: podName,
Expand All @@ -58,12 +55,14 @@ func cmdAdd(args *skel.CmdArgs) error {
NetNs: args.Netns,
IfName: args.IfName,
Provider: netConf.Provider,
Routes: netConf.Routes,
DeviceID: netConf.DeviceID,
VfDriver: netConf.VfDriver,
})
if err != nil {
return err
}

result := generateCNIResult(cniVersion, response)
return types.PrintResult(&result, cniVersion)
}
Expand Down Expand Up @@ -150,9 +149,10 @@ type ipamConf struct {

type netConf struct {
types.NetConf
ServerSocket string `json:"server_socket"`
Provider string `json:"provider"`
IPAM *ipamConf `json:"ipam"`
ServerSocket string `json:"server_socket"`
Provider string `json:"provider"`
Routes []request.Route `json:"routes"`
IPAM *ipamConf `json:"ipam"`
// PciAddrs in case of using sriov
DeviceID string `json:"deviceID"`
VfDriver string `json:"vf_driver"`
Expand Down
41 changes: 39 additions & 2 deletions docs/multi-nic.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,9 +151,9 @@ spec:
image: nginx:alpine
```

# Multi kube-ovn network Interfaces
# Multiple Kube-OVN network Interfaces

Full support for multi kube-ovn networks is more than just IPAM, now the attachment network can also come from OVN.
Full support for multiple Kube-OVN networks is more than just IPAM, now the attachment network can also come from Kube-OVN.

## How to use it

Expand All @@ -180,6 +180,43 @@ spec:

`provider`: The `<name>.<namespace>.ovn` of this NetworkAttachmentDefinition, The kube-OVN plug-in will use it later to determine whether a native OVN subnet should be used. *Be sure to add the OVN suffix*.

In v1.8.0 we introduced support for custom routes. Here is an example:

```yaml
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: attachnet
namespace: default
spec:
config: '{
"cniVersion": "0.3.0",
"type": "kube-ovn",
"server_socket": "/run/openvswitch/kube-ovn-daemon.sock",
"provider": "attachnet.default.ovn",
"routes": [
{
"dst": "19.10.0.0/16"
},
{
"dst": "19.20.0.0/16",
"gw": "19.10.0.1"
}
]
}'
```

The specified routes will be added to the network interface:

```shell
/ # ip route
default via 10.16.0.1 dev eth0
10.16.0.0/16 dev eth0 scope link src 10.16.0.2
19.10.0.0/24 dev net1 scope link src 19.10.0.2
19.10.0.0/16 dev net1
19.20.0.0/16 via 19.10.0.1 dev net1
```

### Create pod with multus ovn network

For random allocation from ovn-default, just add the `k8s.v1.cni.cncf.io/networks`:
Expand Down
6 changes: 3 additions & 3 deletions pkg/daemon/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,12 +173,12 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
mtu = csh.Config.MTU
}

klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s", ifName, macAddr, ipAddr, cidr, gw)
klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, podRequest.Routes)
if nicType == util.InternalType {
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, podRequest.Routes, ingress, egress, vlanID, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
} else {
podNicName = ifName
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, vlanID, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, podRequest.Routes, ingress, egress, vlanID, podRequest.DeviceID, nicType, netns, !podSubnet.Spec.DisableGatewayCheck)
}
if err != nil {
errMsg := fmt.Errorf("configure nic failed %v", err)
Expand Down
112 changes: 68 additions & 44 deletions pkg/daemon/ovs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,11 @@ import (

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/ovs"
"github.com/kubeovn/kube-ovn/pkg/request"
"github.com/kubeovn/kube-ovn/pkg/util"
)

func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway, ingress, egress, vlanID, DeviceID, nicType, podNetns string, checkGw bool) error {
func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns, containerID, vfDriver, ifName, mac string, mtu int, ip, gateway string, routes []request.Route, ingress, egress, vlanID, DeviceID, nicType, podNetns string, checkGw bool) error {
var err error
var hostNicName, containerNicName string
if DeviceID == "" {
Expand Down Expand Up @@ -74,7 +75,7 @@ func (csh cniServerHandler) configureNic(podName, podNamespace, provider, netns,
if err != nil {
return fmt.Errorf("failed to open netns %q: %v", netns, err)
}
if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, mtu, nicType, checkGw); err != nil {
if err = configureContainerNic(containerNicName, ifName, ip, gateway, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
return err
}
return nil
Expand Down Expand Up @@ -157,7 +158,7 @@ func configureHostNic(nicName, vlanID string) error {
return nil
}

func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, checkGw bool) error {
func configureContainerNic(nicName, ifName string, ipAddr, gateway string, routes []request.Route, macAddr net.HardwareAddr, netns ns.NetNS, mtu int, nicType string, checkGw bool) error {
containerLink, err := netlink.LinkByName(nicName)
if err != nil {
return fmt.Errorf("can not find container nic %s %v", nicName, err)
Expand Down Expand Up @@ -206,52 +207,75 @@ func configureContainerNic(nicName, ifName string, ipAddr, gateway string, macAd
}
}

if ifName != "eth0" {
if ifName == "eth0" {
// Only eth0 requires the default route and gateway
return nil
switch util.CheckProtocol(ipAddr) {
case kubeovnv1.ProtocolIPv4:
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gateway),
})
case kubeovnv1.ProtocolIPv6:
_, defaultNet, _ := net.ParseCIDR("::/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gateway),
})
case kubeovnv1.ProtocolDual:
gws := strings.Split(gateway, ",")
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gws[0]),
})
if err != nil {
return fmt.Errorf("config v4 gateway failed %v", err)
}

_, defaultNet, _ = net.ParseCIDR("::/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gws[1]),
})
}

if err != nil {
return fmt.Errorf("failed to configure gateway: %v", err)
}
}

switch util.CheckProtocol(ipAddr) {
case kubeovnv1.ProtocolIPv4:
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gateway),
})
case kubeovnv1.ProtocolIPv6:
_, defaultNet, _ := net.ParseCIDR("::/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gateway),
})
case kubeovnv1.ProtocolDual:
gws := strings.Split(gateway, ",")
_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gws[0]),
})
for _, r := range routes {
_, dst, err := net.ParseCIDR(r.Destination)
if err != nil {
return fmt.Errorf("config v4 gateway failed %v", err)
klog.Errorf("invalid route destination %s: %v", r.Destination, err)
continue
}

_, defaultNet, _ = net.ParseCIDR("::/0")
err = netlink.RouteAdd(&netlink.Route{
LinkIndex: containerLink.Attrs().Index,
Scope: netlink.SCOPE_UNIVERSE,
Dst: defaultNet,
Gw: net.ParseIP(gws[1]),
})
}
var gw net.IP
if r.Gateway != "" {
if gw = net.ParseIP(r.Gateway); gw == nil {
klog.Errorf("invalid route gateway %s", r.Gateway)
continue
}
}

if err != nil {
return fmt.Errorf("config gateway failed %v", err)
route := &netlink.Route{
Dst: dst,
Gw: gw,
LinkIndex: containerLink.Attrs().Index,
}
if err = netlink.RouteReplace(route); err != nil {
klog.Errorf("failed to add route %+v: %v", r, err)
}
}

if checkGw {
Expand Down Expand Up @@ -832,7 +856,7 @@ func renameLink(curName, newName string) error {
return nil
}

func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway, ingress, egress, vlanID, DeviceID, nicType, podNetns string, checkGw bool) (string, error) {
func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace, provider, netns, containerID, ifName, mac string, mtu int, ip, gateway string, routes []request.Route, ingress, egress, vlanID, DeviceID, nicType, podNetns string, checkGw bool) (string, error) {
var err error

_, containerNicName := generateNicName(containerID, ifName)
Expand Down Expand Up @@ -866,7 +890,7 @@ func (csh cniServerHandler) configureNicWithInternalPort(podName, podNamespace,
if err != nil {
return containerNicName, fmt.Errorf("failed to open netns %q: %v", netns, err)
}
if err = configureContainerNic(containerNicName, ifName, ip, gateway, macAddr, podNS, mtu, nicType, checkGw); err != nil {
if err = configureContainerNic(containerNicName, ifName, ip, gateway, routes, macAddr, podNS, mtu, nicType, checkGw); err != nil {
return containerNicName, err
}
return containerNicName, nil
Expand Down
23 changes: 15 additions & 8 deletions pkg/request/cniserver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,23 @@ type CniServerClient struct {
*gorequest.SuperAgent
}

// Route represents a requested route
type Route struct {
Destination string `json:"dst"`
Gateway string `json:"gw"`
}

// CniRequest is the cniserver request format
type CniRequest struct {
CniType string `json:"cni_type"`
PodName string `json:"pod_name"`
PodNamespace string `json:"pod_namespace"`
ContainerID string `json:"container_id"`
NetNs string `json:"net_ns"`
IfName string `json:"if_name"`
Provider string `json:"provider"`
VfDriver string `json:"vf_driver"`
CniType string `json:"cni_type"`
PodName string `json:"pod_name"`
PodNamespace string `json:"pod_namespace"`
ContainerID string `json:"container_id"`
NetNs string `json:"net_ns"`
IfName string `json:"if_name"`
Provider string `json:"provider"`
Routes []Route `json:"routes"`
VfDriver string `json:"vf_driver"`
// PciAddrs in case of using sriov
DeviceID string `json:"deviceID"`
}
Expand Down

0 comments on commit 96b0c11

Please sign in to comment.