From ddbfa67ac78711222ee7cd6f3e64eb5e02fd057d Mon Sep 17 00:00:00 2001 From: Mathieu Rohon Date: Wed, 26 Sep 2018 09:52:37 +0200 Subject: [PATCH] Add portmap capability support Signed-off-by: Mathieu Rohon --- README.md | 35 +++++++++++++------------ examples/multus-ptp-portmap.conf | 36 +++++++++++++++++++++++++ multus/multus.go | 13 ++++----- multus/multus_test.go | 45 ++++++++++++++++++++++++++++++++ types/conf.go | 10 +++++-- types/conf_test.go | 9 ++++++- types/types.go | 24 ++++++++++++----- 7 files changed, 140 insertions(+), 32 deletions(-) create mode 100644 examples/multus-ptp-portmap.conf diff --git a/README.md b/README.md index 1c9bdf273..e9c2d483b 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: name: macvlan-conf -spec: +spec: config: '{ "cniVersion": "0.3.0", "type": "macvlan", @@ -110,7 +110,7 @@ $ kubectl exec -it samplepod -- ip a * CNI configuration stored in on-disk file > refer the section 3.2 Network Object Definition for more details in Kubernetes Network Custom Resource Definition De-facto Standard * Refer the reference implementation presentation and demo details - [link](https://docs.google.com/presentation/d/1dbCin6MnhK-BjjcVun5YiPTL99VA2uSiyWAtWAPNlIc/edit?usp=sharing) -* Release version from v2.0 is not compatible with v1.1 and v1.2 network CRD +* Release version from v2.0 is not compatible with v1.1 and v1.2 network CRD * [MULTUS CNI plugin](#multus-cni-plugin)specifications. ## Multi-Homed pod @@ -137,6 +137,7 @@ $ kubectl exec -it samplepod -- ip a - type (string, required): "multus" - kubeconfig (string, optional): kubeconfig file for the out of cluster communication with kube-apiserver. See the example [kubeconfig](https://github.com/intel/multus-cni/blob/master/doc/node-kubeconfig.yaml) - delegates (([]map,required): number of delegate details in the Multus +- capabilities ({}list, optional): [capabilities](https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md#dynamic-plugin-specific-fields-capabilities--runtime-configuration) supported by at least one of the delegates. (NOTE: Multus only supports portMappings capability for now). See the [example](https://github.com/intel/multus-cni/blob/master/examples/multus-ptp-portmap.conf). ## Usage with Kubernetes CRD based network objects @@ -176,7 +177,7 @@ apiVersion: "k8s.cni.cncf.io/v1" kind: NetworkAttachmentDefinition metadata: name: flannel-networkobj -spec: +spec: config: '{ "cniVersion": "0.3.0", "type": "flannel", @@ -305,7 +306,7 @@ Configurations referenced in annotations are created in addition to the default 1. Save the following YAML to pod-multi-network.yaml. In this case flannel-conf network object acts as the primary network. ``` -# cat pod-multi-network.yaml +# cat pod-multi-network.yaml apiVersion: v1 kind: Pod metadata: @@ -346,7 +347,7 @@ multus-multi-net-poc 1/1 Running 0 30s 1. Run `ifconfig` command in Pod: ``` -# kubectl exec -it multus-multi-net-poc -- ifconfig +# kubectl exec -it multus-multi-net-poc -- ifconfig eth0 Link encap:Ethernet HWaddr C6:43:7C:09:B4:9C inet addr:10.128.0.4 Bcast:0.0.0.0 Mask:255.255.255.0 UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1 @@ -355,39 +356,39 @@ eth0 Link encap:Ethernet HWaddr C6:43:7C:09:B4:9C collisions:0 txqueuelen:0 RX bytes:648 (648.0 B) TX bytes:42 (42.0 B) -lo Link encap:Local Loopback +lo Link encap:Local Loopback inet addr:127.0.0.1 Mask:255.0.0.0 inet6 addr: ::1/128 Scope:Host UP LOOPBACK RUNNING MTU:65536 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:0 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:1 + collisions:0 txqueuelen:1 RX bytes:0 (0.0 B) TX bytes:0 (0.0 B) - -net0 Link encap:Ethernet HWaddr 06:21:91:2D:74:B9 + +net0 Link encap:Ethernet HWaddr 06:21:91:2D:74:B9 inet addr:192.168.42.3 Bcast:0.0.0.0 Mask:255.255.255.0 inet6 addr: fe80::421:91ff:fe2d:74b9/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1450 Metric:1 RX packets:0 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:0 + collisions:0 txqueuelen:0 RX bytes:0 (0.0 B) TX bytes:648 (648.0 B) -net1 Link encap:Ethernet HWaddr D2:94:98:82:00:00 +net1 Link encap:Ethernet HWaddr D2:94:98:82:00:00 inet addr:10.56.217.171 Bcast:0.0.0.0 Mask:255.255.255.0 inet6 addr: fe80::d094:98ff:fe82:0/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:2 errors:0 dropped:0 overruns:0 frame:0 TX packets:8 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:1000 + collisions:0 txqueuelen:1000 RX bytes:120 (120.0 B) TX bytes:648 (648.0 B) -north Link encap:Ethernet HWaddr BE:F2:48:42:83:12 +north Link encap:Ethernet HWaddr BE:F2:48:42:83:12 inet6 addr: fe80::bcf2:48ff:fe42:8312/64 Scope:Link UP BROADCAST RUNNING MULTICAST MTU:1500 Metric:1 RX packets:1420 errors:0 dropped:0 overruns:0 frame:0 TX packets:1276 errors:0 dropped:0 overruns:0 carrier:0 - collisions:0 txqueuelen:1000 + collisions:0 txqueuelen:1000 RX bytes:95956 (93.7 KiB) TX bytes:82200 (80.2 KiB) ``` @@ -424,7 +425,7 @@ Given the following network configuration: { "type": "sriov", #part of sriov plugin conf - "if0": "enp12s0f0", + "if0": "enp12s0f0", "ipam": { "type": "host-local", "subnet": "10.56.217.0/24", @@ -462,7 +463,7 @@ EOF ## Logging Options -You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration. +You may wish to enable some enhanced logging for Multus, especially during the process where you're configuring Multus and need to understand what is or isn't working with your particular configuration. Multus will always log via `STDERR`, which is the standard method by which CNI plugins communicate errors, and these errors are logged by the Kubelet. This method is always enabled. @@ -557,7 +558,7 @@ pod "multus-test" created 20: net0: mtu 1500 qdisc mq qlen 1000 link/ether f6:fb:21:4f:1d:63 brd ff:ff:ff:ff:ff:ff 21: net1: mtu 1500 qdisc mq qlen 1000 - link/ether 76:13:b1:60:00:00 brd ff:ff:ff:ff:ff:ff + link/ether 76:13:b1:60:00:00 brd ff:ff:ff:ff:ff:ff ``` | Interface name | Description | diff --git a/examples/multus-ptp-portmap.conf b/examples/multus-ptp-portmap.conf new file mode 100644 index 000000000..d092c7c53 --- /dev/null +++ b/examples/multus-ptp-portmap.conf @@ -0,0 +1,36 @@ +{ + "name": "multus-cni-network", + "type": "multus" + "capabilities": { + "portMappings": true + }, + "delegates": [ + { + "cniVersion": "0.3.1", + "name": "ptp-tuning-conflist", + "plugins": [ + { + "dns": { + "nameservers": [ + "172.16.1.1" + ] + }, + "ipMasq": true, + "ipam": { + "subnet": "172.16.0.0/24", + "type": "host-local" + }, + "mtu": 512, + "type": "ptp" + }, + { + "capabilities": { + "portMappings": true + }, + "externalSetMarkChain": "KUBE-MARK-MASQ", + "type": "portmap" + } + ] + } + ], +} diff --git a/multus/multus.go b/multus/multus.go index 73f150159..c4ae13aca 100644 --- a/multus/multus.go +++ b/multus/multus.go @@ -111,11 +111,12 @@ func validateIfName(nsname string, ifname string) error { return err } -func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string) (cnitypes.Result, error) { +func conflistAdd(rt *libcni.RuntimeConf, rawnetconflist []byte, binDir string, exec invoke.Exec) (cnitypes.Result, error) { logging.Debugf("conflistAdd: %v, %s, %s", rt, string(rawnetconflist), binDir) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go - binDirs := []string{binDir} - cniNet := libcni.CNIConfig{Path: binDirs} + binDirs := filepath.SplitList(os.Getenv("CNI_PATH")) + binDirs = append(binDirs, binDir) + cniNet := libcni.NewCNIConfig(binDirs, exec) confList, err := libcni.ConfListFromBytes(rawnetconflist) if err != nil { @@ -160,7 +161,7 @@ func delegateAdd(exec invoke.Exec, ifName string, delegate *types.DelegateNetCon } if delegate.ConfListPlugin != false { - result, err := conflistAdd(rt, delegate.Bytes, binDir) + result, err := conflistAdd(rt, delegate.Bytes, binDir, exec) if err != nil { return nil, logging.Errorf("Multus: error in invoke Conflist add - %q: %v", delegate.ConfList.Name, err) } @@ -246,7 +247,7 @@ func cmdAdd(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) (cn for idx, delegate := range n.Delegates { lastIdx = idx ifName := getIfname(delegate, args.IfName, idx) - rt, _ = types.LoadCNIRuntimeConf(args, k8sArgs, ifName) + rt, _ = types.LoadCNIRuntimeConf(args, k8sArgs, ifName, n.RuntimeConfig) tmpResult, err = delegateAdd(exec, ifName, delegate, rt, n.BinDir) if err != nil { break @@ -360,7 +361,7 @@ func cmdDel(args *skel.CmdArgs, exec invoke.Exec, kubeClient k8s.KubeClient) err } } - rt, _ := types.LoadCNIRuntimeConf(args, k8sArgs, "") + rt, _ := types.LoadCNIRuntimeConf(args, k8sArgs, "", in.RuntimeConfig) return delPlugins(exec, args.IfName, in.Delegates, len(in.Delegates)-1, rt, in.BinDir) } diff --git a/multus/multus_test.go b/multus/multus_test.go index 35cad0255..2aa46927d 100644 --- a/multus/multus_test.go +++ b/multus/multus_test.go @@ -306,4 +306,49 @@ var _ = Describe("multus operations", func() { // plugin 1 is the masterplugin Expect(reflect.DeepEqual(r, expectedResult1)).To(BeTrue()) }) + + It("ensure delegates get portmap runtime config", func() { + args := &skel.CmdArgs{ + ContainerID: "123456789", + Netns: testNS.Path(), + IfName: "eth0", + StdinData: []byte(`{ + "name": "node-cni-network", + "type": "multus", + "delegates": [{ + "cniVersion": "0.3.1", + "name": "mynet-confList", + "plugins": [ + { + "type": "firstPlugin", + "capabilities": {"portMappings": true} + } + ] + }], + "runtimeConfig": { + "portMappings": [ + {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"} + ] + } +}`), + } + + fExec := &fakeExec{} + expectedConf1 := `{ + "capabilities": {"portMappings": true}, + "name": "mynet-confList", + "cniVersion": "0.3.1", + "type": "firstPlugin", + "runtimeConfig": { + "portMappings": [ + {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"} + ] + } +}` + fExec.addPlugin(nil, "eth0", expectedConf1, nil, nil) + os.Setenv("CNI_COMMAND", "ADD") + os.Setenv("CNI_IFNAME", "eth0") + _, err := cmdAdd(args, fExec, nil) + Expect(err).NotTo(HaveOccurred()) + }) }) diff --git a/types/conf.go b/types/conf.go index e95d9cee2..010fe5be8 100644 --- a/types/conf.go +++ b/types/conf.go @@ -73,9 +73,9 @@ func LoadDelegateNetConf(bytes []byte, ifnameRequest string) (*DelegateNetConf, return delegateConf, nil } -func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*libcni.RuntimeConf, error) { +func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string, rc *RuntimeConfig) (*libcni.RuntimeConf, error) { - logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s", args, k8sArgs, ifName) + logging.Debugf("LoadCNIRuntimeConf: %v, %v, %s, %v", args, k8sArgs, ifName, rc) // In part, adapted from K8s pkg/kubelet/dockershim/network/cni/cni.go#buildCNIRuntimeConf // Todo // ingress, egress and bandwidth capability features as same as kubelet. @@ -90,6 +90,12 @@ func LoadCNIRuntimeConf(args *skel.CmdArgs, k8sArgs *K8sArgs, ifName string) (*l {"K8S_POD_INFRA_CONTAINER_ID", string(k8sArgs.K8S_POD_INFRA_CONTAINER_ID)}, }, } + + if rc != nil { + rt.CapabilityArgs = map[string]interface{}{ + "portMappings": rc.PortMaps, + } + } return rt, nil } diff --git a/types/conf_test.go b/types/conf_test.go index fa64a3ddb..c69eeb4ea 100644 --- a/types/conf_test.go +++ b/types/conf_test.go @@ -35,13 +35,20 @@ var _ = Describe("config operations", func() { "kubeconfig": "/etc/kubernetes/node-kubeconfig.yaml", "delegates": [{ "type": "weave-net" - }] + }], + "runtimeConfig": { + "portMappings": [ + {"hostPort": 8080, "containerPort": 80, "protocol": "tcp"} + ] + } + }` netConf, err := LoadNetConf([]byte(conf)) Expect(err).NotTo(HaveOccurred()) Expect(len(netConf.Delegates)).To(Equal(1)) Expect(netConf.Delegates[0].Conf.Type).To(Equal("weave-net")) Expect(netConf.Delegates[0].MasterPlugin).To(BeTrue()) + Expect(len(netConf.RuntimeConfig.PortMaps)).To(Equal(1)) }) It("succeeds if only delegates are set", func() { diff --git a/types/types.go b/types/types.go index 20ed055ea..4570904b0 100644 --- a/types/types.go +++ b/types/types.go @@ -36,12 +36,24 @@ type NetConf struct { CNIDir string `json:"cniDir"` BinDir string `json:"binDir"` // RawDelegates is private to the NetConf class; use Delegates instead - RawDelegates []map[string]interface{} `json:"delegates"` - Delegates []*DelegateNetConf `json:"-"` - NetStatus []*NetworkStatus `json:"-"` - Kubeconfig string `json:"kubeconfig"` - LogFile string `json:"logFile"` - LogLevel string `json:"logLevel"` + RawDelegates []map[string]interface{} `json:"delegates"` + Delegates []*DelegateNetConf `json:"-"` + NetStatus []*NetworkStatus `json:"-"` + Kubeconfig string `json:"kubeconfig"` + LogFile string `json:"logFile"` + LogLevel string `json:"logLevel"` + RuntimeConfig *RuntimeConfig `json:"runtimeConfig,omitempty"` +} + +type RuntimeConfig struct { + PortMaps []PortMapEntry `json:"portMappings,omitempty"` +} + +type PortMapEntry struct { + HostPort int `json:"hostPort"` + ContainerPort int `json:"containerPort"` + Protocol string `json:"protocol"` + HostIP string `json:"hostIP,omitempty"` } type NetworkStatus struct {