From b78e535055300c9091889eec7344ffb4d66e334b Mon Sep 17 00:00:00 2001 From: DennisDenuto Date: Sun, 7 Jan 2018 19:15:17 -0800 Subject: [PATCH 1/8] plugins/meta/bandwith: traffic shaping plugin Add chained plugin to add a tbf qdisc to shape ingress/egress traffic --- pkg/testutils/echosvr/echosvr_test.go | 2 +- pkg/testutils/echosvr/main.go | 27 +- plugins/linux_only.txt | 1 + plugins/meta/bandwidth/README.md | 64 ++ .../meta/bandwidth/bandwidth_linux_test.go | 565 ++++++++++++++++++ .../meta/bandwidth/bandwidth_suite_test.go | 199 ++++++ plugins/meta/bandwidth/ifb_creator.go | 165 +++++ plugins/meta/bandwidth/main.go | 218 +++++++ 8 files changed, 1236 insertions(+), 5 deletions(-) create mode 100644 plugins/meta/bandwidth/README.md create mode 100644 plugins/meta/bandwidth/bandwidth_linux_test.go create mode 100644 plugins/meta/bandwidth/bandwidth_suite_test.go create mode 100644 plugins/meta/bandwidth/ifb_creator.go create mode 100644 plugins/meta/bandwidth/main.go diff --git a/pkg/testutils/echosvr/echosvr_test.go b/pkg/testutils/echosvr/echosvr_test.go index 901febd0c8d8..cca1fffb1b1e 100644 --- a/pkg/testutils/echosvr/echosvr_test.go +++ b/pkg/testutils/echosvr/echosvr_test.go @@ -68,7 +68,7 @@ var _ = Describe("Echosvr", func() { Expect(err).NotTo(HaveOccurred()) defer conn.Close() - fmt.Fprintf(conn, "hello") + fmt.Fprintf(conn, "hello\n") Expect(ioutil.ReadAll(conn)).To(Equal([]byte("hello"))) }) }) diff --git a/pkg/testutils/echosvr/main.go b/pkg/testutils/echosvr/main.go index 7e932bfcdf00..fe6ac46ffd8b 100644 --- a/pkg/testutils/echosvr/main.go +++ b/pkg/testutils/echosvr/main.go @@ -7,8 +7,13 @@ package main import ( + "bufio" "fmt" + "io" "net" + "os" + "strings" + "time" ) func main() { @@ -31,8 +36,22 @@ func main() { } func handleConnection(conn net.Conn) { - buf := make([]byte, 512) - nBytesRead, _ := conn.Read(buf) - conn.Write(buf[0:nBytesRead]) - conn.Close() + conn.SetReadDeadline(time.Now().Add(1 * time.Minute)) + content, err := bufio.NewReader(conn).ReadString('\n') + if err != nil && err != io.EOF { + fmt.Fprint(os.Stderr, err.Error()) + return + } + + conn.SetWriteDeadline(time.Now().Add(1 * time.Minute)) + if _, err = conn.Write([]byte(strings.TrimSuffix(content, "\n"))); err != nil { + fmt.Fprint(os.Stderr, err.Error()) + return + } + + if err = conn.Close(); err != nil { + fmt.Fprint(os.Stderr, err.Error()) + return + } + } diff --git a/plugins/linux_only.txt b/plugins/linux_only.txt index c644e20fd27c..6b04296f547c 100644 --- a/plugins/linux_only.txt +++ b/plugins/linux_only.txt @@ -8,3 +8,4 @@ plugins/main/ptp plugins/main/vlan plugins/meta/portmap plugins/meta/tuning +plugins/meta/bandwidth \ No newline at end of file diff --git a/plugins/meta/bandwidth/README.md b/plugins/meta/bandwidth/README.md new file mode 100644 index 000000000000..7129055ea620 --- /dev/null +++ b/plugins/meta/bandwidth/README.md @@ -0,0 +1,64 @@ +# bandwidth plugin + +## Overview + +This plugin provides a way to use and configure Linux's Traffic control (tc) subystem. tc encompasses the sets of mechanisms and operations by which packets are queued for transmission/reception on a network interface. + +This plugin configures a token bucket filter (tbf) queuing discipline (qdisc) on both ingress and egress traffic. Resulting in traffic being shaped when reading / writing. + +Due to limitations on tc shaping rules for ingress, this plugin creates an Intermediate Functional Block device (ifb) to redirect packets from the host interface. tc tbf is then applied to the ifb device. The packets that were redirected to the ifb devices, are written OUT (and shaped) to the host interface. + +This plugin is only useful when used in addition to other plugins. + +## Chaining + +The bandwidth plugin applies traffic shaping to interfaces (as described above) created by previously applied plugins. + +The following is an example [json configuration list](https://github.com/containernetworking/cni/blob/master/SPEC.md#network-configuration-list-runtime-examples) for creating a `ptp` between the host -> container via veth interfaces, whereby traffic is shaped by the `bandwidth` plugin: + +```json +{ + "cniVersion": "0.3.1", + "name": "mynet", + "plugins": [ + { + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.0.0.0/24" + }, + "dns": { + "nameservers": [ "10.1.0.1" ] + } + }, + { + "name": "slowdown", + "type": "bandwidth", + "ingressRate": 123, + "ingressBurst": 456, + "egressRate": 123, + "egressBurst": 456 + } + ] +} +``` + +The result is an `ifb` device in the host namespace redirecting to the `host-interface`, with `tc tbf` applied on the `ifb` device and the `container-interface` + +## Network configuration reference +* ingressRate: is the rate in Kbps at which traffic can enter an interface. (See http://man7.org/linux/man-pages/man8/tbf.8.html) +* ingressBurst: is the maximum amount in Kb that tokens can be made available for instantaneously. (See http://man7.org/linux/man-pages/man8/tbf.8.html) +* egressRate: is the rate in Kbps at which traffic can leave an interface. (See http://man7.org/linux/man-pages/man8/tbf.8.html) +* egressBurst: is the maximum amount in Kb that tokens can be made available for instantaneously. (See http://man7.org/linux/man-pages/man8/tbf.8.html) + +Both ingressRate and ingressBurst must be set in order to limit ingress bandwidth. If neither one is set, then ingress bandwidth is not limited. +Both egressRate and egressBurst must be set in order to limit egress bandwidth. If neither one is set, then egress bandwidth is not limited. + + +## tc tbf documentation + +- [tldp traffic control](http://tldp.org/HOWTO/Traffic-Control-HOWTO/components.html) +- [man tbf](http://man7.org/linux/man-pages/man8/tbf.8.html) +- [tc ingress and ifb mirroring](https://serverfault.com/questions/350023/tc-ingress-policing-and-ifb-mirroring) diff --git a/plugins/meta/bandwidth/bandwidth_linux_test.go b/plugins/meta/bandwidth/bandwidth_linux_test.go new file mode 100644 index 000000000000..fdc30f06db4e --- /dev/null +++ b/plugins/meta/bandwidth/bandwidth_linux_test.go @@ -0,0 +1,565 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bandwidth + +import ( + "fmt" + + "encoding/json" + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containernetworking/plugins/pkg/testutils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + "github.com/vishvananda/netlink" + "net" + "time" +) + +var _ = Describe("bandwidth test", func() { + var ( + hostNs ns.NetNS + containerNs ns.NetNS + ifbDeviceName string + hostIfname string + containerIfname string + hostIP net.IP + containerIP net.IP + hostIfaceMTU int + ) + + BeforeEach(func() { + var err error + + hostIfname = "host-veth" + containerIfname = "container-veth" + + hostNs, err = ns.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerNs, err = ns.NewNS() + Expect(err).NotTo(HaveOccurred()) + + hostIP = net.IP{169, 254, 0, 1} + containerIP = net.IP{10, 254, 0, 1} + hostIfaceMTU = 1024 + ifbDeviceName = "5b6c" + + createVeth(hostNs.Path(), hostIfname, containerNs.Path(), containerIfname, hostIP, containerIP, hostIfaceMTU) + }) + + AfterEach(func() { + containerNs.Close() + hostNs.Close() + }) + + Describe("cmdADD", func() { + It("Works with a Veth pair using 0.3.0 config", func() { + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 8, + "ingressBurst": 8, + "egressRate": 16, + "egressBurst": 9, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } +}` + + conf = fmt.Sprintf(conf, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + r, out, err := testutils.CmdAddWithResult(containerNs.Path(), "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + result, err := current.GetResult(r) + Expect(err).NotTo(HaveOccurred()) + + Expect(result.Interfaces).To(HaveLen(3)) + Expect(result.Interfaces[2].Name).To(Equal(ifbDeviceName)) + Expect(result.Interfaces[2].Sandbox).To(Equal("")) + + ifbLink, err := netlink.LinkByName(ifbDeviceName) + Expect(err).NotTo(HaveOccurred()) + Expect(ifbLink.Attrs().MTU).To(Equal(hostIfaceMTU)) + + qdiscs, err := netlink.QdiscList(ifbLink) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscs).To(HaveLen(1)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) + + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(2))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(9))) + + hostVethLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) + + qdiscFilters, err := netlink.FilterList(hostVethLink, netlink.MakeHandle(0xffff, 0)) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscFilters).To(HaveLen(1)) + Expect(qdiscFilters[0].(*netlink.U32).Actions[0].(*netlink.MirredAction).Ifindex).To(Equal(ifbLink.Attrs().Index)) + + return nil + })).To(Succeed()) + + Expect(hostNs.Do(func(n ns.NetNS) error { + defer GinkgoRecover() + + ifbLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) + + qdiscs, err := netlink.QdiscList(ifbLink) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscs).To(HaveLen(2)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(ifbLink.Attrs().Index)) + + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(1))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(8))) + return nil + })).To(Succeed()) + + }) + + It("Does not apply ingress when disabled", func() { + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 0, + "egressRate": 8, + "egressBurst": 1, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } +}` + + conf = fmt.Sprintf(conf, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, out, err := testutils.CmdAddWithResult(containerNs.Path(), ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + _, err = netlink.LinkByName(ifbDeviceName) + Expect(err).NotTo(HaveOccurred()) + return nil + })).To(Succeed()) + + Expect(hostNs.Do(func(n ns.NetNS) error { + defer GinkgoRecover() + + containerIfLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) + + qdiscs, err := netlink.QdiscList(containerIfLink) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscs).To(HaveLen(2)) + Expect(qdiscs[0]).NotTo(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[1]).NotTo(BeAssignableToTypeOf(&netlink.Tbf{})) + + return nil + })).To(Succeed()) + + }) + + It("Does not apply egress when disabled", func() { + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "egressRate": 0, + "egressBurst": 0, + "ingressRate": 8, + "ingressBurst": 1, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } +}` + + conf = fmt.Sprintf(conf, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, out, err := testutils.CmdAddWithResult(containerNs.Path(), ifbDeviceName, []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + _, err = netlink.LinkByName(ifbDeviceName) + Expect(err).To(HaveOccurred()) + return nil + })).To(Succeed()) + + Expect(hostNs.Do(func(n ns.NetNS) error { + defer GinkgoRecover() + + containerIfLink, err := netlink.LinkByName(hostIfname) + Expect(err).NotTo(HaveOccurred()) + + qdiscs, err := netlink.QdiscList(containerIfLink) + Expect(err).NotTo(HaveOccurred()) + + Expect(qdiscs).To(HaveLen(1)) + Expect(qdiscs[0].Attrs().LinkIndex).To(Equal(containerIfLink.Attrs().Index)) + + Expect(qdiscs[0]).To(BeAssignableToTypeOf(&netlink.Tbf{})) + Expect(qdiscs[0].(*netlink.Tbf).Rate).To(Equal(uint64(1))) + Expect(qdiscs[0].(*netlink.Tbf).Limit).To(Equal(uint32(1))) + return nil + })).To(Succeed()) + + }) + + It("fails an invalid ingress config", func() { + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 0, + "ingressBurst": 123, + "egressRate": 123, + "egressBurst": 123, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } +}` + + conf = fmt.Sprintf(conf, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: "eth0", + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + + _, _, err := testutils.CmdAddWithResult(containerNs.Path(), "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).To(MatchError("if burst is set, rate must also be set")) + return nil + })).To(Succeed()) + }) + }) + + Describe("cmdDEL", func() { + It("Works with a Veth pair using 0.3.0 config", func() { + conf := `{ + "cniVersion": "0.3.0", + "name": "cni-plugin-bandwidth-test", + "type": "bandwidth", + "ingressRate": 8, + "ingressBurst": 8, + "egressRate": 9, + "egressBurst": 9, + "prevResult": { + "interfaces": [ + { + "name": "%s", + "sandbox": "" + }, + { + "name": "%s", + "sandbox": "%s" + } + ], + "ips": [ + { + "version": "4", + "address": "%s/24", + "gateway": "10.0.0.1", + "interface": 1 + } + ], + "routes": [] + } +}` + + conf = fmt.Sprintf(conf, hostIfname, containerIfname, containerNs.Path(), containerIP.String()) + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerNs.Path(), + IfName: containerIfname, + StdinData: []byte(conf), + } + + Expect(hostNs.Do(func(netNS ns.NetNS) error { + defer GinkgoRecover() + _, out, err := testutils.CmdAddWithResult(containerNs.Path(), "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + err = testutils.CmdDelWithResult(containerNs.Path(), "", func() error { return cmdDel(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + _, err = netlink.LinkByName(ifbDeviceName) + Expect(err).To(HaveOccurred()) + + return nil + })).To(Succeed()) + + }) + }) + + Context("when chaining bandwidth plugin with PTP using 0.3.0 config", func() { + var ptpConf string + var rateInBits int + var burstInBits int + var packetInBytes int + var containerWithoutTbfNS ns.NetNS + var containerWithTbfNS ns.NetNS + var portServerWithTbf int + var portServerWithoutTbf int + + var containerWithTbfRes types.Result + var containerWithoutTbfRes types.Result + var echoServerWithTbf *gexec.Session + var echoServerWithoutTbf *gexec.Session + + BeforeEach(func() { + rateInBytes := 1000 + rateInBits = rateInBytes * 8 + burstInBits = rateInBits * 2 + packetInBytes = rateInBytes * 25 + + ptpConf = `{ + "cniVersion": "0.3.0", + "name": "mynet", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24" + } +}` + + containerWithTbfIFName := "ptp0" + containerWithoutTbfIFName := "ptp1" + + var err error + containerWithTbfNS, err = ns.NewNS() + Expect(err).NotTo(HaveOccurred()) + + containerWithoutTbfNS, err = ns.NewNS() + Expect(err).NotTo(HaveOccurred()) + + By("create two containers, and use the bandwidth plugin on one of them") + Expect(hostNs.Do(func(ns.NetNS) error { + defer GinkgoRecover() + + containerWithTbfRes, _, err = testutils.CmdAddWithResult(containerWithTbfNS.Path(), containerWithTbfIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd("ptp", []byte(ptpConf)) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithoutTbfRes, _, err = testutils.CmdAddWithResult(containerWithoutTbfNS.Path(), containerWithoutTbfIFName, []byte(ptpConf), func() error { + r, err := invoke.DelegateAdd("ptp", []byte(ptpConf)) + Expect(r.Print()).To(Succeed()) + + return err + }) + Expect(err).NotTo(HaveOccurred()) + + containerWithTbfResult, err := current.GetResult(containerWithTbfRes) + Expect(err).NotTo(HaveOccurred()) + + tbfPluginConf := PluginConf{ + IngressBurst: burstInBits, + IngressRate: rateInBits, + EgressBurst: burstInBits, + EgressRate: rateInBits, + } + tbfPluginConf.Name = "mynet" + tbfPluginConf.CNIVersion = "0.3.0" + tbfPluginConf.Type = "bandwidth" + tbfPluginConf.RawPrevResult = &map[string]interface{}{ + "ips": containerWithTbfResult.IPs, + "interfaces": containerWithTbfResult.Interfaces, + } + + tbfPluginConf.PrevResult = ¤t.Result{ + IPs: containerWithTbfResult.IPs, + Interfaces: containerWithTbfResult.Interfaces, + } + + conf, err := json.Marshal(tbfPluginConf) + Expect(err).NotTo(HaveOccurred()) + + args := &skel.CmdArgs{ + ContainerID: "dummy", + Netns: containerWithTbfNS.Path(), + IfName: containerWithTbfIFName, + StdinData: []byte(conf), + } + + _, out, err := testutils.CmdAddWithResult(containerWithTbfNS.Path(), "", []byte(conf), func() error { return cmdAdd(args) }) + Expect(err).NotTo(HaveOccurred(), string(out)) + + return nil + })).To(Succeed()) + + By("starting a tcp server on both containers") + portServerWithTbf, echoServerWithTbf, err = startEchoServerInNamespace(containerWithTbfNS) + Expect(err).NotTo(HaveOccurred()) + portServerWithoutTbf, echoServerWithoutTbf, err = startEchoServerInNamespace(containerWithoutTbfNS) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + containerWithTbfNS.Close() + containerWithoutTbfNS.Close() + if echoServerWithoutTbf != nil { + echoServerWithoutTbf.Kill() + } + if echoServerWithTbf != nil { + echoServerWithTbf.Kill() + } + }) + + Measure("limits ingress traffic on veth device", func(b Benchmarker) { + var runtimeWithLimit time.Duration + var runtimeWithoutLimit time.Duration + + By("gather timing statistics about both containers") + By("sending tcp traffic to the container that has traffic shaped", func() { + runtimeWithLimit = b.Time("with tbf", func() { + result, err := current.GetResult(containerWithTbfRes) + Expect(err).NotTo(HaveOccurred()) + + makeTcpClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithTbf, packetInBytes) + }) + }) + + By("sending tcp traffic to the container that does not have traffic shaped", func() { + runtimeWithoutLimit = b.Time("without tbf", func() { + result, err := current.GetResult(containerWithoutTbfRes) + Expect(err).NotTo(HaveOccurred()) + + makeTcpClientInNS(hostNs.Path(), result.IPs[0].Address.IP.String(), portServerWithoutTbf, packetInBytes) + }) + }) + + Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) + }, 1) + }) + +}) diff --git a/plugins/meta/bandwidth/bandwidth_suite_test.go b/plugins/meta/bandwidth/bandwidth_suite_test.go new file mode 100644 index 000000000000..7a191cdb46ed --- /dev/null +++ b/plugins/meta/bandwidth/bandwidth_suite_test.go @@ -0,0 +1,199 @@ +package bandwidth + +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "bytes" + "fmt" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" + "github.com/vishvananda/netlink" + "io" + "net" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +func TestTBF(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "bandwidth suite") +} + +var echoServerBinaryPath string + +var _ = SynchronizedBeforeSuite(func() []byte { + binaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echosvr") + Expect(err).NotTo(HaveOccurred()) + return []byte(binaryPath) +}, func(data []byte) { + echoServerBinaryPath = string(data) +}) + +var _ = SynchronizedAfterSuite(func() {}, func() { + gexec.CleanupBuildArtifacts() +}) + +func startInNetNS(binPath string, netNS ns.NetNS) (*gexec.Session, error) { + baseName := filepath.Base(netNS.Path()) + // we are relying on the netNS path living in /var/run/netns + // where `ip netns exec` can find it + cmd := exec.Command("ip", "netns", "exec", baseName, binPath) + + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + return session, err +} + +func startEchoServerInNamespace(netNS ns.NetNS) (int, *gexec.Session, error) { + session, err := startInNetNS(echoServerBinaryPath, netNS) + Expect(err).NotTo(HaveOccurred()) + + // wait for it to print it's address on stdout + Eventually(session.Out).Should(gbytes.Say("\n")) + _, portString, err := net.SplitHostPort(strings.TrimSpace(string(session.Out.Contents()))) + Expect(err).NotTo(HaveOccurred()) + + port, err := strconv.Atoi(portString) + Expect(err).NotTo(HaveOccurred()) + + go func() { + // print out echoserver output to ginkgo to capture any errors that might be occurring. + io.Copy(GinkgoWriter, io.MultiReader(session.Out, session.Err)) + }() + + return port, session, nil +} + +func makeTcpClientInNS(netns string, address string, port int, numBytes int) { + message := bytes.Repeat([]byte{'a'}, numBytes) + + bin, err := exec.LookPath("nc") + Expect(err).NotTo(HaveOccurred()) + var cmd *exec.Cmd + if netns != "" { + netns = filepath.Base(netns) + cmd = exec.Command("ip", "netns", "exec", netns, bin, "-v", address, strconv.Itoa(port)) + } else { + cmd = exec.Command("nc", address, strconv.Itoa(port)) + } + cmd.Stdin = bytes.NewBuffer([]byte(message)) + cmd.Stderr = GinkgoWriter + out, err := cmd.Output() + + Expect(err).NotTo(HaveOccurred()) + Expect(string(out)).To(Equal(string(message))) +} + +func createVeth(hostNamespace string, hostVethIfName string, containerNamespace string, containerVethIfName string, hostIP []byte, containerIP []byte, hostIfaceMTU int) { + vethDeviceRequest := &netlink.Veth{ + LinkAttrs: netlink.LinkAttrs{ + Name: hostVethIfName, + Flags: net.FlagUp, + MTU: hostIfaceMTU, + }, + PeerName: containerVethIfName, + } + + hostNs, err := ns.GetNS(hostNamespace) + Expect(err).NotTo(HaveOccurred()) + + err = hostNs.Do(func(_ ns.NetNS) error { + if err := netlink.LinkAdd(vethDeviceRequest); err != nil { + return fmt.Errorf("creating veth pair: %s", err) + } + + containerVeth, err := netlink.LinkByName(containerVethIfName) + if err != nil { + return fmt.Errorf("failed to find newly-created veth device %q: %v", containerVethIfName, err) + } + + containerNs, err := ns.GetNS(containerNamespace) + if err != nil { + return err + } + + err = netlink.LinkSetNsFd(containerVeth, int(containerNs.Fd())) + if err != nil { + return fmt.Errorf("failed to move veth to container namespace: %s", err) + } + + localAddr := &net.IPNet{ + IP: hostIP, + Mask: []byte{255, 255, 255, 255}, + } + peerAddr := &net.IPNet{ + IP: containerIP, + Mask: []byte{255, 255, 255, 255}, + } + addr, err := netlink.ParseAddr(localAddr.String()) + if err != nil { + return fmt.Errorf("parsing address %s: %s", localAddr, err) + } + + addr.Peer = peerAddr + + addr.Scope = int(netlink.SCOPE_LINK) + hostVeth, err := netlink.LinkByName(hostVethIfName) + if err != nil { + return fmt.Errorf("failed to find newly-created veth device %q: %v", containerVethIfName, err) + } + + err = netlink.AddrAdd(hostVeth, addr) + if err != nil { + return fmt.Errorf("adding IP address %s: %s", localAddr, err) + } + + return nil + }) + Expect(err).NotTo(HaveOccurred()) + + containerNs, err := ns.GetNS(containerNamespace) + err = containerNs.Do(func(_ ns.NetNS) error { + peerAddr := &net.IPNet{ + IP: hostIP, + Mask: []byte{255, 255, 255, 255}, + } + localAddr := &net.IPNet{ + IP: containerIP, + Mask: []byte{255, 255, 255, 255}, + } + addr, err := netlink.ParseAddr(localAddr.String()) + if err != nil { + return fmt.Errorf("parsing address %s: %s", localAddr, err) + } + + addr.Peer = peerAddr + + addr.Scope = int(netlink.SCOPE_LINK) + containerVeth, err := netlink.LinkByName(containerVethIfName) + if err != nil { + return fmt.Errorf("failed to find newly-created veth device %q: %v", containerVethIfName, err) + } + err = netlink.AddrAdd(containerVeth, addr) + if err != nil { + return fmt.Errorf("adding IP address %s: %s", localAddr, err) + } + + return nil + }) + + Expect(err).NotTo(HaveOccurred()) +} diff --git a/plugins/meta/bandwidth/ifb_creator.go b/plugins/meta/bandwidth/ifb_creator.go new file mode 100644 index 000000000000..190aa88f9987 --- /dev/null +++ b/plugins/meta/bandwidth/ifb_creator.go @@ -0,0 +1,165 @@ +package bandwidth + +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import ( + "fmt" + "github.com/containernetworking/plugins/pkg/ip" + "github.com/vishvananda/netlink" + "net" + "syscall" +) + +const latencyInMillis = 25 + +func CreateIfb(ifbDeviceName string, mtu int) error { + err := netlink.LinkAdd(&netlink.Ifb{ + LinkAttrs: netlink.LinkAttrs{ + Name: ifbDeviceName, + Flags: net.FlagUp, + MTU: mtu, + }, + }) + + if err != nil { + return fmt.Errorf("adding link: %s", err) + } + + return nil +} + +func TeardownIfb(deviceName string) error { + _, err := ip.DelLinkByNameAddr(deviceName) + if err != nil && err == ip.ErrLinkNotFound { + return nil + } + return err +} + +func CreateIngressQdisc(rateInBits, burstInBits int, hostDeviceName string) error { + hostDevice, err := netlink.LinkByName(hostDeviceName) + if err != nil { + return fmt.Errorf("get host device: %s", err) + } + return createTBF(rateInBits, burstInBits, hostDevice.Attrs().Index) +} + +func CreateEgressQdisc(rateInBits, burstInBits int, hostDeviceName string, ifbDeviceName string) error { + ifbDevice, err := netlink.LinkByName(ifbDeviceName) + if err != nil { + return fmt.Errorf("get ifb device: %s", err) + } + hostDevice, err := netlink.LinkByName(hostDeviceName) + if err != nil { + return fmt.Errorf("get host device: %s", err) + } + + // add qdisc ingress on host device + ingress := &netlink.Ingress{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: hostDevice.Attrs().Index, + Handle: netlink.MakeHandle(0xffff, 0), // ffff: + Parent: netlink.HANDLE_INGRESS, + }, + } + + err = netlink.QdiscAdd(ingress) + if err != nil { + return fmt.Errorf("create ingress qdisc: %s", err) + } + + // add filter on host device to mirror traffic to ifb device + filter := &netlink.U32{ + FilterAttrs: netlink.FilterAttrs{ + LinkIndex: hostDevice.Attrs().Index, + Parent: ingress.QdiscAttrs.Handle, + Priority: 1, + Protocol: syscall.ETH_P_ALL, + }, + ClassId: netlink.MakeHandle(1, 1), + RedirIndex: ifbDevice.Attrs().Index, + Actions: []netlink.Action{ + &netlink.MirredAction{ + ActionAttrs: netlink.ActionAttrs{}, + MirredAction: netlink.TCA_EGRESS_REDIR, + Ifindex: ifbDevice.Attrs().Index, + }, + }, + } + err = netlink.FilterAdd(filter) + if err != nil { + return fmt.Errorf("add filter: %s", err) + } + + // throttle traffic on ifb device + err = createTBF(rateInBits, burstInBits, ifbDevice.Attrs().Index) + if err != nil { + return fmt.Errorf("create ifb qdisc: %s", err) + } + return nil +} + +func createTBF(rateInBits, burstInBits, linkIndex int) error { + // Equivalent to + // tc qdisc add dev link root tbf + // rate netConf.BandwidthLimits.Rate + // burst netConf.BandwidthLimits.Burst + if rateInBits <= 0 { + return fmt.Errorf("invalid rate: %d", rateInBits) + } + if burstInBits <= 0 { + return fmt.Errorf("invalid burst: %d", burstInBits) + } + rateInBytes := rateInBits / 8 + bufferInBytes := buffer(uint64(rateInBytes), uint32(burstInBits)) + latency := latencyInUsec(latencyInMillis) + limitInBytes := limit(uint64(rateInBytes), latency, uint32(bufferInBytes)) + + qdisc := &netlink.Tbf{ + QdiscAttrs: netlink.QdiscAttrs{ + LinkIndex: linkIndex, + Handle: netlink.MakeHandle(1, 0), + Parent: netlink.HANDLE_ROOT, + }, + Limit: uint32(limitInBytes), + Rate: uint64(rateInBytes), + Buffer: uint32(bufferInBytes), + } + err := netlink.QdiscAdd(qdisc) + if err != nil { + return fmt.Errorf("create qdisc: %s", err) + } + return nil +} + +func tick2Time(tick uint32) uint32 { + return uint32(float64(tick) / float64(netlink.TickInUsec())) +} + +func time2Tick(time uint32) uint32 { + return uint32(float64(time) * float64(netlink.TickInUsec())) +} + +func buffer(rate uint64, burst uint32) uint32 { + return time2Tick(uint32(float64(burst) * float64(netlink.TIME_UNITS_PER_SEC) / float64(rate))) +} + +func limit(rate uint64, latency float64, buffer uint32) uint32 { + return uint32(float64(rate) / float64(netlink.TIME_UNITS_PER_SEC) * (latency + float64(tick2Time(buffer)))) +} + +func latencyInUsec(latencyInMillis float64) float64 { + return float64(netlink.TIME_UNITS_PER_SEC) * (latencyInMillis / 1000.0) +} diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go new file mode 100644 index 000000000000..60e18709b58b --- /dev/null +++ b/plugins/meta/bandwidth/main.go @@ -0,0 +1,218 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package bandwidth + +import ( + "crypto/sha1" + "encoding/json" + "errors" + "fmt" + "github.com/containernetworking/cni/pkg/skel" + "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/cni/pkg/version" + "github.com/vishvananda/netlink" +) + +type PluginConf struct { + types.NetConf + RuntimeConfig *struct{} `json:"runtimeConfig"` + + // This is the previous result, when called in the context of a chained + // plugin. Because this plugin supports multiple versions, we'll have to + // parse this in two passes. If your plugin is not chained, this can be + // removed (though you may wish to error if a non-chainable plugin is + // chained. + // If you need to modify the result before returning it, you will need + // to actually convert it to a concrete versioned struct. + RawPrevResult *map[string]interface{} `json:"prevResult"` + PrevResult *current.Result `json:"-"` + + // Add plugin-specifc flags here + IngressRate int `json:"ingressRate"` //Bandwidth rate in Kbps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set + IngressBurst int `json:"ingressBurst"` //Bandwidth burst in Kb for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set + + EgressRate int `json:"egressRate"` //Bandwidth rate in Kbps for traffic through container. 0 for no limit. If egressRate is set, egressBurst must also be set + EgressBurst int `json:"egressBurst"` //Bandwidth burst in Kb for traffic through container. 0 for no limit. If egressBurst is set, egressRate must also be set +} + +// parseConfig parses the supplied configuration (and prevResult) from stdin. +func parseConfig(stdin []byte) (*PluginConf, error) { + conf := PluginConf{} + + if err := json.Unmarshal(stdin, &conf); err != nil { + return nil, fmt.Errorf("failed to parse network configuration: %v", err) + } + + // Parse previous result. Remove this if your plugin is not chained. + if conf.RawPrevResult != nil { + resultBytes, err := json.Marshal(conf.RawPrevResult) + if err != nil { + return nil, fmt.Errorf("could not serialize prevResult: %v", err) + } + res, err := version.NewResult(conf.CNIVersion, resultBytes) + if err != nil { + return nil, fmt.Errorf("could not parse prevResult: %v", err) + } + conf.RawPrevResult = nil + conf.PrevResult, err = current.NewResultFromResult(res) + if err != nil { + return nil, fmt.Errorf("could not convert result to current version: %v", err) + } + } + // End previous result parsing + + err := validateRateAndBurst(conf.IngressRate, conf.IngressBurst) + if err != nil { + return nil, err + } + err = validateRateAndBurst(conf.EgressRate, conf.EgressBurst) + if err != nil { + return nil, err + } + + return &conf, nil + +} + +func validateRateAndBurst(rate int, burst int) error { + switch { + case burst < 0 || rate < 0: + return fmt.Errorf("rate and burst must be a positive integer") + case burst == 0 && rate != 0: + return fmt.Errorf("if rate is set, burst must also be set") + case rate == 0 && burst != 0: + return fmt.Errorf("if burst is set, rate must also be set") + } + + return nil +} + +func getIfbDeviceName(networkName string, containerId string) (string, error) { + hash := sha1.New() + _, err := hash.Write([]byte(networkName + containerId)) + if err != nil { + return "", err + } + + return fmt.Sprintf("%x", hash.Sum(nil))[:4], nil +} + +func getMTU(deviceName string) (int, error) { + link, err := netlink.LinkByName(deviceName) + if err != nil { + return -1, err + } + + return link.Attrs().MTU, nil +} + +func getHostInterface(interfaces []*current.Interface) (*current.Interface, error) { + for _, prevIface := range interfaces { + if prevIface.Sandbox != "" { + continue + } + + return prevIface, nil + } + + return nil, errors.New("no host interface found") +} + +// cmdAdd is called for ADD requests +func cmdAdd(args *skel.CmdArgs) error { + conf, err := parseConfig(args.StdinData) + if err != nil { + return err + } + + //no traffic shaping was requested, so just no-op and quit + if conf.IngressRate == 0 && conf.IngressBurst == 0 && conf.EgressRate == 0 && conf.EgressBurst == 0 { + return types.PrintResult(conf.PrevResult, conf.CNIVersion) + } + + if conf.PrevResult == nil { + return fmt.Errorf("must be called as chained plugin") + } + + hostInterface, err := getHostInterface(conf.PrevResult.Interfaces) + if err != nil { + return err + } + + if conf.IngressRate > 0 && conf.IngressBurst > 0 { + err = CreateIngressQdisc(conf.IngressRate, conf.IngressBurst, hostInterface.Name) + if err != nil { + return err + } + } + + if conf.EgressRate > 0 && conf.EgressBurst > 0 { + mtu, err := getMTU(hostInterface.Name) + if err != nil { + return err + } + + ifbDeviceName, err := getIfbDeviceName(conf.Name, args.ContainerID) + if err != nil { + return err + } + + err = CreateIfb(ifbDeviceName, mtu) + if err != nil { + return err + } + + ifbDevice, err := netlink.LinkByName(ifbDeviceName) + if err != nil { + return err + } + + conf.PrevResult.Interfaces = append(conf.PrevResult.Interfaces, ¤t.Interface{ + Name: ifbDeviceName, + Mac: ifbDevice.Attrs().HardwareAddr.String(), + }) + err = CreateEgressQdisc(conf.EgressRate, conf.EgressBurst, hostInterface.Name, ifbDeviceName) + if err != nil { + return err + } + } + + // Pass through the result for the next plugin + return types.PrintResult(conf.PrevResult, conf.CNIVersion) +} + +// cmdDel is called for DELETE requests +func cmdDel(args *skel.CmdArgs) error { + conf, err := parseConfig(args.StdinData) + if err != nil { + return err + } + + ifbDeviceName, err := getIfbDeviceName(conf.Name, args.ContainerID) + if err != nil { + return err + } + + if err := TeardownIfb(ifbDeviceName); err != nil { + return err + } + + return nil +} + +func main() { + skel.PluginMain(cmdAdd, cmdDel, version.PluginSupports("0.3.0", "0.3.1", version.Current())) +} From dce91d11d6a44e5f2ffa0954ee44c9be1b41b48c Mon Sep 17 00:00:00 2001 From: Gabe Rosenhouse Date: Sat, 17 Feb 2018 15:17:02 -0800 Subject: [PATCH 2/8] meta/bandwidth: remove boilerplate comments --- plugins/meta/bandwidth/main.go | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go index 60e18709b58b..13e4160de663 100644 --- a/plugins/meta/bandwidth/main.go +++ b/plugins/meta/bandwidth/main.go @@ -19,6 +19,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" @@ -30,17 +31,9 @@ type PluginConf struct { types.NetConf RuntimeConfig *struct{} `json:"runtimeConfig"` - // This is the previous result, when called in the context of a chained - // plugin. Because this plugin supports multiple versions, we'll have to - // parse this in two passes. If your plugin is not chained, this can be - // removed (though you may wish to error if a non-chainable plugin is - // chained. - // If you need to modify the result before returning it, you will need - // to actually convert it to a concrete versioned struct. RawPrevResult *map[string]interface{} `json:"prevResult"` PrevResult *current.Result `json:"-"` - // Add plugin-specifc flags here IngressRate int `json:"ingressRate"` //Bandwidth rate in Kbps for traffic through container. 0 for no limit. If ingressRate is set, ingressBurst must also be set IngressBurst int `json:"ingressBurst"` //Bandwidth burst in Kb for traffic through container. 0 for no limit. If ingressBurst is set, ingressRate must also be set @@ -56,7 +49,6 @@ func parseConfig(stdin []byte) (*PluginConf, error) { return nil, fmt.Errorf("failed to parse network configuration: %v", err) } - // Parse previous result. Remove this if your plugin is not chained. if conf.RawPrevResult != nil { resultBytes, err := json.Marshal(conf.RawPrevResult) if err != nil { @@ -72,7 +64,6 @@ func parseConfig(stdin []byte) (*PluginConf, error) { return nil, fmt.Errorf("could not convert result to current version: %v", err) } } - // End previous result parsing err := validateRateAndBurst(conf.IngressRate, conf.IngressBurst) if err != nil { @@ -131,7 +122,6 @@ func getHostInterface(interfaces []*current.Interface) (*current.Interface, erro return nil, errors.New("no host interface found") } -// cmdAdd is called for ADD requests func cmdAdd(args *skel.CmdArgs) error { conf, err := parseConfig(args.StdinData) if err != nil { @@ -190,11 +180,9 @@ func cmdAdd(args *skel.CmdArgs) error { } } - // Pass through the result for the next plugin return types.PrintResult(conf.PrevResult, conf.CNIVersion) } -// cmdDel is called for DELETE requests func cmdDel(args *skel.CmdArgs) error { conf, err := parseConfig(args.StdinData) if err != nil { From 59fa37252f6531510bef26ec900ffb7052d06be1 Mon Sep 17 00:00:00 2001 From: Gabe Rosenhouse Date: Sat, 17 Feb 2018 15:19:31 -0800 Subject: [PATCH 3/8] meta/bandwidth: group and sort imports ref: https://github.com/golang/go/wiki/CodeReviewComments#imports --- plugins/meta/bandwidth/bandwidth_linux_test.go | 10 ++++++---- plugins/meta/bandwidth/bandwidth_suite_test.go | 16 +++++++++------- plugins/meta/bandwidth/ifb_creator.go | 6 ++++-- plugins/meta/bandwidth/main.go | 1 + 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/plugins/meta/bandwidth/bandwidth_linux_test.go b/plugins/meta/bandwidth/bandwidth_linux_test.go index fdc30f06db4e..d1e50cf2a943 100644 --- a/plugins/meta/bandwidth/bandwidth_linux_test.go +++ b/plugins/meta/bandwidth/bandwidth_linux_test.go @@ -15,21 +15,23 @@ package bandwidth import ( + "encoding/json" "fmt" + "net" + "time" + + "github.com/vishvananda/netlink" - "encoding/json" "github.com/containernetworking/cni/pkg/invoke" "github.com/containernetworking/cni/pkg/skel" "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/plugins/pkg/ns" "github.com/containernetworking/plugins/pkg/testutils" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" - "github.com/vishvananda/netlink" - "net" - "time" ) var _ = Describe("bandwidth test", func() { diff --git a/plugins/meta/bandwidth/bandwidth_suite_test.go b/plugins/meta/bandwidth/bandwidth_suite_test.go index 7a191cdb46ed..c96d6c9e23a1 100644 --- a/plugins/meta/bandwidth/bandwidth_suite_test.go +++ b/plugins/meta/bandwidth/bandwidth_suite_test.go @@ -15,15 +15,8 @@ package bandwidth // limitations under the License. import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "bytes" "fmt" - "github.com/containernetworking/plugins/pkg/ns" - "github.com/onsi/gomega/gbytes" - "github.com/onsi/gomega/gexec" - "github.com/vishvananda/netlink" "io" "net" "os/exec" @@ -31,6 +24,15 @@ import ( "strconv" "strings" "testing" + + "github.com/containernetworking/plugins/pkg/ns" + "github.com/onsi/gomega/gbytes" + "github.com/onsi/gomega/gexec" + + "github.com/vishvananda/netlink" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) func TestTBF(t *testing.T) { diff --git a/plugins/meta/bandwidth/ifb_creator.go b/plugins/meta/bandwidth/ifb_creator.go index 190aa88f9987..48e6b094e6c6 100644 --- a/plugins/meta/bandwidth/ifb_creator.go +++ b/plugins/meta/bandwidth/ifb_creator.go @@ -16,10 +16,12 @@ package bandwidth import ( "fmt" - "github.com/containernetworking/plugins/pkg/ip" - "github.com/vishvananda/netlink" "net" "syscall" + + "github.com/containernetworking/plugins/pkg/ip" + + "github.com/vishvananda/netlink" ) const latencyInMillis = 25 diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go index 13e4160de663..828846438e68 100644 --- a/plugins/meta/bandwidth/main.go +++ b/plugins/meta/bandwidth/main.go @@ -24,6 +24,7 @@ import ( "github.com/containernetworking/cni/pkg/types" "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" + "github.com/vishvananda/netlink" ) From 90252c30fbb8072cd01925eed690208c6ba29842 Mon Sep 17 00:00:00 2001 From: Gabe Rosenhouse Date: Tue, 27 Feb 2018 22:20:50 -0800 Subject: [PATCH 4/8] meta/bandwidth: package main so we can build a binary --- plugins/meta/bandwidth/bandwidth_linux_test.go | 2 +- plugins/meta/bandwidth/bandwidth_suite_test.go | 3 +-- plugins/meta/bandwidth/ifb_creator.go | 4 ++-- plugins/meta/bandwidth/main.go | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/plugins/meta/bandwidth/bandwidth_linux_test.go b/plugins/meta/bandwidth/bandwidth_linux_test.go index d1e50cf2a943..befc6f6765d9 100644 --- a/plugins/meta/bandwidth/bandwidth_linux_test.go +++ b/plugins/meta/bandwidth/bandwidth_linux_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bandwidth +package main import ( "encoding/json" diff --git a/plugins/meta/bandwidth/bandwidth_suite_test.go b/plugins/meta/bandwidth/bandwidth_suite_test.go index c96d6c9e23a1..3af4427171b4 100644 --- a/plugins/meta/bandwidth/bandwidth_suite_test.go +++ b/plugins/meta/bandwidth/bandwidth_suite_test.go @@ -1,5 +1,3 @@ -package bandwidth - // Copyright 2018 CNI authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,6 +11,7 @@ package bandwidth // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. +package main import ( "bytes" diff --git a/plugins/meta/bandwidth/ifb_creator.go b/plugins/meta/bandwidth/ifb_creator.go index 48e6b094e6c6..a7032c0e894c 100644 --- a/plugins/meta/bandwidth/ifb_creator.go +++ b/plugins/meta/bandwidth/ifb_creator.go @@ -1,5 +1,3 @@ -package bandwidth - // Copyright 2018 CNI authors // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -14,6 +12,8 @@ package bandwidth // See the License for the specific language governing permissions and // limitations under the License. +package main + import ( "fmt" "net" diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go index 828846438e68..9835eef3f63c 100644 --- a/plugins/meta/bandwidth/main.go +++ b/plugins/meta/bandwidth/main.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package bandwidth +package main import ( "crypto/sha1" From 2793dd11cb5c3d826bd90458063257a8092a4141 Mon Sep 17 00:00:00 2001 From: Gabe Rosenhouse Date: Wed, 28 Feb 2018 08:59:04 -0800 Subject: [PATCH 5/8] top-level integration test coverage for ptp + bandwidth --- .travis.yml | 1 + integration/integration_linux_test.go | 88 +++++++++++++++++++ integration/integration_suite_test.go | 32 +++++++ integration/testdata/basic-ptp.json | 11 +++ .../testdata/chained-ptp-bandwidth.conflist | 22 +++++ 5 files changed, 154 insertions(+) create mode 100644 integration/integration_linux_test.go create mode 100644 integration/integration_suite_test.go create mode 100644 integration/testdata/basic-ptp.json create mode 100644 integration/testdata/chained-ptp-bandwidth.conflist diff --git a/.travis.yml b/.travis.yml index 525a9847d5ba..e6ab0e99ee93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,7 @@ matrix: install: - go get github.com/onsi/ginkgo/ginkgo + - go get github.com/containernetworking/cni/cnitool script: - | diff --git a/integration/integration_linux_test.go b/integration/integration_linux_test.go new file mode 100644 index 000000000000..7a8ea4512740 --- /dev/null +++ b/integration/integration_linux_test.go @@ -0,0 +1,88 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package integration_test + +import ( + "fmt" + "math/rand" + "os" + "os/exec" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" +) + +var _ = Describe("Basic PTP using cnitool", func() { + var ( + env TestEnv + nsShortName string + nsLongName string + cnitoolBin string + ) + + BeforeEach(func() { + cniPath, err := filepath.Abs("../bin") + Expect(err).NotTo(HaveOccurred()) + netConfPath, err := filepath.Abs("./testdata") + Expect(err).NotTo(HaveOccurred()) + cnitoolBin, err = exec.LookPath("cnitool") + Expect(err).NotTo(HaveOccurred(), "expected to find cnitool in your PATH") + + env = TestEnv([]string{ + "CNI_PATH=" + cniPath, + "NETCONFPATH=" + netConfPath, + "PATH=" + os.Getenv("PATH"), + }) + + nsShortName = fmt.Sprintf("cni-test-%x", rand.Int31()) + nsLongName = fmt.Sprintf("/var/run/netns/" + nsShortName) + }) + + It("supports basic network add and del operations", func() { + env.run("ip", "netns", "add", nsShortName) + defer env.run("ip", "netns", "del", nsShortName) + + env.run(cnitoolBin, "add", "basic-ptp", nsLongName) + + addrOutput := env.run("ip", "netns", "exec", nsShortName, "ip", "addr") + Expect(addrOutput).To(ContainSubstring("10.1.2.")) + + env.run(cnitoolBin, "del", "basic-ptp", nsLongName) + }) + + It("supports add and del with chained plugins", func() { + env.run("ip", "netns", "add", nsShortName) + defer env.run("ip", "netns", "del", nsShortName) + + env.run(cnitoolBin, "add", "chained-ptp-bandwidth", nsLongName) + + addrOutput := env.run("ip", "netns", "exec", nsShortName, "ip", "addr") + Expect(addrOutput).To(ContainSubstring("10.9.2.")) + + env.run(cnitoolBin, "del", "chained-ptp-bandwidth", nsLongName) + }) +}) + +type TestEnv []string + +func (e TestEnv) run(bin string, args ...string) string { + cmd := exec.Command(bin, args...) + cmd.Env = e + session, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter) + Expect(err).NotTo(HaveOccurred()) + Eventually(session, "5s").Should(gexec.Exit(0)) + return string(session.Out.Contents()) +} diff --git a/integration/integration_suite_test.go b/integration/integration_suite_test.go new file mode 100644 index 000000000000..740b9fdb4cfb --- /dev/null +++ b/integration/integration_suite_test.go @@ -0,0 +1,32 @@ +// Copyright 2018 CNI authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package integration_test + +import ( + "math/rand" + "testing" + + . "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/config" + . "github.com/onsi/gomega" +) + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Integration Suite") +} + +var _ = BeforeSuite(func() { + rand.Seed(config.GinkgoConfig.RandomSeed + int64(GinkgoParallelNode())) +}) diff --git a/integration/testdata/basic-ptp.json b/integration/testdata/basic-ptp.json new file mode 100644 index 000000000000..11dc3aa61981 --- /dev/null +++ b/integration/testdata/basic-ptp.json @@ -0,0 +1,11 @@ +{ + "cniVersion": "0.3.0", + "name": "basic-ptp", + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.1.2.0/24" + } +} diff --git a/integration/testdata/chained-ptp-bandwidth.conflist b/integration/testdata/chained-ptp-bandwidth.conflist new file mode 100644 index 000000000000..3919da0f1a5a --- /dev/null +++ b/integration/testdata/chained-ptp-bandwidth.conflist @@ -0,0 +1,22 @@ +{ + "cniVersion": "0.3.1", + "name": "chained-ptp-bandwidth", + "plugins": [ + { + "type": "ptp", + "ipMasq": true, + "mtu": 512, + "ipam": { + "type": "host-local", + "subnet": "10.9.2.0/24" + } + }, + { + "type": "bandwidth", + "ingressRate": 800, + "ingressBurst": 200, + "egressRate": 800, + "egressBurst": 200 + } + ] +} From d5bdfe4cbdc47ce85f23c28aa3664d7d44906046 Mon Sep 17 00:00:00 2001 From: Gabe Rosenhouse Date: Wed, 28 Feb 2018 13:28:06 -0800 Subject: [PATCH 6/8] top-level integration test for bridge + bandwidth --- integration/integration_linux_test.go | 68 +++++++++++++------ .../chained-bridge-bandwidth.conflist | 20 ++++++ 2 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 integration/testdata/chained-bridge-bandwidth.conflist diff --git a/integration/integration_linux_test.go b/integration/integration_linux_test.go index 7a8ea4512740..6c23adb1ad20 100644 --- a/integration/integration_linux_test.go +++ b/integration/integration_linux_test.go @@ -27,10 +27,10 @@ import ( var _ = Describe("Basic PTP using cnitool", func() { var ( - env TestEnv - nsShortName string - nsLongName string - cnitoolBin string + env TestEnv + hostNS NSShortName + contNS NSShortName + cnitoolBin string ) BeforeEach(func() { @@ -47,32 +47,37 @@ var _ = Describe("Basic PTP using cnitool", func() { "PATH=" + os.Getenv("PATH"), }) - nsShortName = fmt.Sprintf("cni-test-%x", rand.Int31()) - nsLongName = fmt.Sprintf("/var/run/netns/" + nsShortName) - }) + hostNS = NSShortName(fmt.Sprintf("cni-test-host-%x", rand.Int31())) + hostNS.Add() - It("supports basic network add and del operations", func() { - env.run("ip", "netns", "add", nsShortName) - defer env.run("ip", "netns", "del", nsShortName) + contNS = NSShortName(fmt.Sprintf("cni-test-cont-%x", rand.Int31())) + contNS.Add() + }) - env.run(cnitoolBin, "add", "basic-ptp", nsLongName) + AfterEach(func() { + contNS.Del() + hostNS.Del() + }) - addrOutput := env.run("ip", "netns", "exec", nsShortName, "ip", "addr") - Expect(addrOutput).To(ContainSubstring("10.1.2.")) + basicAssertion := func(netName, expectedIPPrefix string) { + env.runInNS(hostNS, cnitoolBin, "add", netName, contNS.LongName()) - env.run(cnitoolBin, "del", "basic-ptp", nsLongName) - }) + addrOutput := env.runInNS(contNS, "ip", "addr") + Expect(addrOutput).To(ContainSubstring(expectedIPPrefix)) - It("supports add and del with chained plugins", func() { - env.run("ip", "netns", "add", nsShortName) - defer env.run("ip", "netns", "del", nsShortName) + env.runInNS(hostNS, cnitoolBin, "del", netName, contNS.LongName()) + } - env.run(cnitoolBin, "add", "chained-ptp-bandwidth", nsLongName) + It("supports basic network add and del operations", func() { + basicAssertion("basic-ptp", "10.1.2.") + }) - addrOutput := env.run("ip", "netns", "exec", nsShortName, "ip", "addr") - Expect(addrOutput).To(ContainSubstring("10.9.2.")) + It("supports add and del with ptp + bandwidth", func() { + basicAssertion("chained-ptp-bandwidth", "10.9.2.") + }) - env.run(cnitoolBin, "del", "chained-ptp-bandwidth", nsLongName) + It("supports add and del with bridge + bandwidth", func() { + basicAssertion("chained-bridge-bandwidth", "10.11.2.") }) }) @@ -86,3 +91,22 @@ func (e TestEnv) run(bin string, args ...string) string { Eventually(session, "5s").Should(gexec.Exit(0)) return string(session.Out.Contents()) } + +func (e TestEnv) runInNS(nsShortName NSShortName, bin string, args ...string) string { + a := append([]string{"netns", "exec", string(nsShortName), bin}, args...) + return e.run("ip", a...) +} + +type NSShortName string + +func (n NSShortName) LongName() string { + return fmt.Sprintf("/var/run/netns/%s", n) +} + +func (n NSShortName) Add() { + (TestEnv{}).run("ip", "netns", "add", string(n)) +} + +func (n NSShortName) Del() { + (TestEnv{}).run("ip", "netns", "del", string(n)) +} diff --git a/integration/testdata/chained-bridge-bandwidth.conflist b/integration/testdata/chained-bridge-bandwidth.conflist new file mode 100644 index 000000000000..b406a649a0db --- /dev/null +++ b/integration/testdata/chained-bridge-bandwidth.conflist @@ -0,0 +1,20 @@ +{ + "cniVersion": "0.3.1", + "name": "chained-bridge-bandwidth", + "plugins": [ + { + "type": "bridge", + "ipam": { + "type": "host-local", + "subnet": "10.11.2.0/24" + } + }, + { + "type": "bandwidth", + "ingressRate": 800, + "ingressBurst": 200, + "egressRate": 800, + "egressBurst": 200 + } + ] +} From d2f6472474deacf4b40721233026c7e998513cb7 Mon Sep 17 00:00:00 2001 From: Aidan Obley Date: Mon, 12 Mar 2018 15:08:53 -0700 Subject: [PATCH 7/8] Ensure the bandwith plugin chooses the host veth device When chained with a plugin that returns multiple devices, the bandwidth plugin chooses the host veth device. Signed-off-by: Tyler Schultz --- integration/integration_linux_test.go | 240 +++++++++++++++--- integration/integration_suite_test.go | 14 +- .../basic-bridge/network-chain-test.json | 12 + .../chained-bridge-bandwidth.conflist | 20 -- .../network-chain-test.conflist | 23 ++ plugins/meta/bandwidth/main.go | 18 +- 6 files changed, 261 insertions(+), 66 deletions(-) create mode 100644 integration/testdata/basic-bridge/network-chain-test.json delete mode 100644 integration/testdata/chained-bridge-bandwidth.conflist create mode 100644 integration/testdata/chained-bridge-bandwidth/network-chain-test.conflist diff --git a/integration/integration_linux_test.go b/integration/integration_linux_test.go index 6c23adb1ad20..e593f76cb3b2 100644 --- a/integration/integration_linux_test.go +++ b/integration/integration_linux_test.go @@ -20,64 +20,181 @@ import ( "os/exec" "path/filepath" + "bytes" + "io" + "net" + "regexp" + "strconv" + "strings" + "time" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" ) var _ = Describe("Basic PTP using cnitool", func() { var ( - env TestEnv - hostNS NSShortName - contNS NSShortName cnitoolBin string + cniPath string ) BeforeEach(func() { - cniPath, err := filepath.Abs("../bin") - Expect(err).NotTo(HaveOccurred()) - netConfPath, err := filepath.Abs("./testdata") + var err error + cniPath, err = filepath.Abs("../bin") Expect(err).NotTo(HaveOccurred()) cnitoolBin, err = exec.LookPath("cnitool") Expect(err).NotTo(HaveOccurred(), "expected to find cnitool in your PATH") + }) + + Context("basic cases", func() { + var ( + env TestEnv + hostNS Namespace + contNS Namespace + ) + + BeforeEach(func() { + var err error + + netConfPath, err := filepath.Abs("./testdata") + Expect(err).NotTo(HaveOccurred()) + + env = TestEnv([]string{ + "CNI_PATH=" + cniPath, + "NETCONFPATH=" + netConfPath, + "PATH=" + os.Getenv("PATH"), + }) - env = TestEnv([]string{ - "CNI_PATH=" + cniPath, - "NETCONFPATH=" + netConfPath, - "PATH=" + os.Getenv("PATH"), + hostNS = Namespace(fmt.Sprintf("cni-test-host-%x", rand.Int31())) + hostNS.Add() + + contNS = Namespace(fmt.Sprintf("cni-test-cont-%x", rand.Int31())) + contNS.Add() + }) + + AfterEach(func() { + contNS.Del() + hostNS.Del() }) - hostNS = NSShortName(fmt.Sprintf("cni-test-host-%x", rand.Int31())) - hostNS.Add() + basicAssertion := func(netName, expectedIPPrefix string) { + env.runInNS(hostNS, cnitoolBin, "add", netName, contNS.LongName()) - contNS = NSShortName(fmt.Sprintf("cni-test-cont-%x", rand.Int31())) - contNS.Add() - }) + addrOutput := env.runInNS(contNS, "ip", "addr") + Expect(addrOutput).To(ContainSubstring(expectedIPPrefix)) + + env.runInNS(hostNS, cnitoolBin, "del", netName, contNS.LongName()) + } + + It("supports basic network add and del operations", func() { + basicAssertion("basic-ptp", "10.1.2.") + }) - AfterEach(func() { - contNS.Del() - hostNS.Del() + It("supports add and del with ptp + bandwidth", func() { + basicAssertion("chained-ptp-bandwidth", "10.9.2.") + }) }) - basicAssertion := func(netName, expectedIPPrefix string) { - env.runInNS(hostNS, cnitoolBin, "add", netName, contNS.LongName()) + Context("when the bandwidth plugin is chained with a plugin that returns multiple adapters", func() { + var ( + hostNS Namespace + contNS1 Namespace + contNS2 Namespace + basicBridgeEnv TestEnv + chainedBridgeBandwidthEnv TestEnv + chainedBridgeBandwidthSession, basicBridgeSession *gexec.Session + chainedBridgeBandwidthPort, basicBridgePort int + chainedBridgeIP, basicBridgeIP string + runtimeWithLimit, runtimeWithoutLimit time.Duration + ) - addrOutput := env.runInNS(contNS, "ip", "addr") - Expect(addrOutput).To(ContainSubstring(expectedIPPrefix)) + BeforeEach(func() { + hostNS = Namespace(fmt.Sprintf("cni-test-host-%x", rand.Int31())) + hostNS.Add() - env.runInNS(hostNS, cnitoolBin, "del", netName, contNS.LongName()) - } + contNS1 = Namespace(fmt.Sprintf("cni-test-cont1-%x", rand.Int31())) + contNS1.Add() - It("supports basic network add and del operations", func() { - basicAssertion("basic-ptp", "10.1.2.") - }) + contNS2 = Namespace(fmt.Sprintf("cni-test-cont2-%x", rand.Int31())) + contNS2.Add() - It("supports add and del with ptp + bandwidth", func() { - basicAssertion("chained-ptp-bandwidth", "10.9.2.") - }) + basicBridgeNetConfPath, err := filepath.Abs("./testdata/basic-bridge") + Expect(err).NotTo(HaveOccurred()) + + basicBridgeEnv = TestEnv([]string{ + "CNI_PATH=" + cniPath, + "NETCONFPATH=" + basicBridgeNetConfPath, + "PATH=" + os.Getenv("PATH"), + }) + + chainedBridgeBandwidthNetConfPath, err := filepath.Abs("./testdata/chained-bridge-bandwidth") + Expect(err).NotTo(HaveOccurred()) + + chainedBridgeBandwidthEnv = TestEnv([]string{ + "CNI_PATH=" + cniPath, + "NETCONFPATH=" + chainedBridgeBandwidthNetConfPath, + "PATH=" + os.Getenv("PATH"), + }) + }) + + AfterEach(func() { + if chainedBridgeBandwidthSession != nil { + chainedBridgeBandwidthSession.Kill() + } + if basicBridgeSession != nil { + basicBridgeSession.Kill() + } + + chainedBridgeBandwidthEnv.runInNS(hostNS, cnitoolBin, "del", "network-chain-test", contNS1.LongName()) + basicBridgeEnv.runInNS(hostNS, cnitoolBin, "del", "network-chain-test", contNS2.LongName()) + }) + + Measure("limits traffic only on the restricted bandwith veth device", func(b Benchmarker) { + ipRegexp := regexp.MustCompile("10\\.11\\.2\\.\\d{1,3}") + + By(fmt.Sprintf("adding %s to %s\n\n", "chained-bridge-bandwidth", contNS1.ShortName()), func() { + chainedBridgeBandwidthEnv.runInNS(hostNS, cnitoolBin, "add", "network-chain-test", contNS1.LongName()) + chainedBridgeIP = ipRegexp.FindString(chainedBridgeBandwidthEnv.runInNS(contNS1, "ip", "addr")) + Expect(chainedBridgeIP).To(ContainSubstring("10.11.2.")) + }) - It("supports add and del with bridge + bandwidth", func() { - basicAssertion("chained-bridge-bandwidth", "10.11.2.") + By(fmt.Sprintf("adding %s to %s\n\n", "basic-bridge", contNS2.ShortName()), func() { + basicBridgeEnv.runInNS(hostNS, cnitoolBin, "add", "network-chain-test", contNS2.LongName()) + basicBridgeIP = ipRegexp.FindString(basicBridgeEnv.runInNS(contNS2, "ip", "addr")) + Expect(basicBridgeIP).To(ContainSubstring("10.11.2.")) + }) + + var err error + + By(fmt.Sprintf("starting echo server in %s\n\n", contNS1.ShortName()), func() { + chainedBridgeBandwidthPort, chainedBridgeBandwidthSession, err = startEchoServerInNamespace(contNS1) + Expect(err).ToNot(HaveOccurred()) + }) + + By(fmt.Sprintf("starting echo server in %s\n\n", contNS2.ShortName()), func() { + basicBridgePort, basicBridgeSession, err = startEchoServerInNamespace(contNS2) + Expect(err).ToNot(HaveOccurred()) + }) + + packetInBytes := 20000 // The shaper needs to 'warm'. Send enough to cause it to throttle, + // balanced by run time. + + By(fmt.Sprintf("sending tcp traffic to the chained, bridged, traffic shaped container on ip address '%s:%d'\n\n", chainedBridgeIP, chainedBridgeBandwidthPort), func() { + runtimeWithLimit = b.Time("with chained bridge and bandwidth plugins", func() { + makeTcpClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes) + }) + }) + + By(fmt.Sprintf("sending tcp traffic to the basic bridged container on ip address '%s:%d'\n\n", basicBridgeIP, basicBridgePort), func() { + runtimeWithoutLimit = b.Time("with basic bridged plugin", func() { + makeTcpClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes) + }) + }) + + Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) + }, 1) }) }) @@ -92,21 +209,70 @@ func (e TestEnv) run(bin string, args ...string) string { return string(session.Out.Contents()) } -func (e TestEnv) runInNS(nsShortName NSShortName, bin string, args ...string) string { +func (e TestEnv) runInNS(nsShortName Namespace, bin string, args ...string) string { a := append([]string{"netns", "exec", string(nsShortName), bin}, args...) return e.run("ip", a...) } -type NSShortName string +type Namespace string -func (n NSShortName) LongName() string { +func (n Namespace) LongName() string { return fmt.Sprintf("/var/run/netns/%s", n) } -func (n NSShortName) Add() { +func (n Namespace) ShortName() string { + return string(n) +} + +func (n Namespace) Add() { (TestEnv{}).run("ip", "netns", "add", string(n)) } -func (n NSShortName) Del() { +func (n Namespace) Del() { (TestEnv{}).run("ip", "netns", "del", string(n)) } + +func makeTcpClientInNS(netns string, address string, port int, numBytes int) { + message := bytes.Repeat([]byte{'a'}, numBytes) + + bin, err := exec.LookPath("nc") + Expect(err).NotTo(HaveOccurred()) + var cmd *exec.Cmd + if netns != "" { + netns = filepath.Base(netns) + cmd = exec.Command("ip", "netns", "exec", netns, bin, "-v", address, strconv.Itoa(port)) + } else { + cmd = exec.Command("nc", address, strconv.Itoa(port)) + } + cmd.Stdin = bytes.NewBuffer([]byte(message)) + cmd.Stderr = GinkgoWriter + out, err := cmd.Output() + + Expect(err).NotTo(HaveOccurred()) + Expect(string(out)).To(Equal(string(message))) +} + +func startEchoServerInNamespace(netNS Namespace) (int, *gexec.Session, error) { + session, err := startInNetNS(echoServerBinaryPath, netNS) + Expect(err).NotTo(HaveOccurred()) + + // wait for it to print it's address on stdout + Eventually(session.Out).Should(gbytes.Say("\n")) + _, portString, err := net.SplitHostPort(strings.TrimSpace(string(session.Out.Contents()))) + Expect(err).NotTo(HaveOccurred()) + + port, err := strconv.Atoi(portString) + Expect(err).NotTo(HaveOccurred()) + + go func() { + // print out echoserver output to ginkgo to capture any errors that might be occurring. + io.Copy(GinkgoWriter, io.MultiReader(session.Out, session.Err)) + }() + + return port, session, nil +} + +func startInNetNS(binPath string, namespace Namespace) (*gexec.Session, error) { + cmd := exec.Command("ip", "netns", "exec", namespace.ShortName(), binPath) + return gexec.Start(cmd, GinkgoWriter, GinkgoWriter) +} diff --git a/integration/integration_suite_test.go b/integration/integration_suite_test.go index 740b9fdb4cfb..d30ea5b3176e 100644 --- a/integration/integration_suite_test.go +++ b/integration/integration_suite_test.go @@ -20,6 +20,7 @@ import ( . "github.com/onsi/ginkgo" "github.com/onsi/ginkgo/config" . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" ) func TestIntegration(t *testing.T) { @@ -27,6 +28,17 @@ func TestIntegration(t *testing.T) { RunSpecs(t, "Integration Suite") } -var _ = BeforeSuite(func() { +var echoServerBinaryPath string + +var _ = SynchronizedBeforeSuite(func() []byte { + binaryPath, err := gexec.Build("github.com/containernetworking/plugins/pkg/testutils/echosvr") + Expect(err).NotTo(HaveOccurred()) + return []byte(binaryPath) +}, func(data []byte) { + echoServerBinaryPath = string(data) rand.Seed(config.GinkgoConfig.RandomSeed + int64(GinkgoParallelNode())) }) + +var _ = SynchronizedAfterSuite(func() {}, func() { + gexec.CleanupBuildArtifacts() +}) diff --git a/integration/testdata/basic-bridge/network-chain-test.json b/integration/testdata/basic-bridge/network-chain-test.json new file mode 100644 index 000000000000..0232eb42c2cc --- /dev/null +++ b/integration/testdata/basic-bridge/network-chain-test.json @@ -0,0 +1,12 @@ +{ + "cniVersion": "0.3.1", + "name": "network-chain-test", + "type": "bridge", + "bridge": "test-bridge-0", + "isDefaultGateway": true, + "ipam": { + "type": "host-local", + "subnet": "10.11.2.0/24", + "dataDir": "/tmp/foo" + } +} diff --git a/integration/testdata/chained-bridge-bandwidth.conflist b/integration/testdata/chained-bridge-bandwidth.conflist deleted file mode 100644 index b406a649a0db..000000000000 --- a/integration/testdata/chained-bridge-bandwidth.conflist +++ /dev/null @@ -1,20 +0,0 @@ -{ - "cniVersion": "0.3.1", - "name": "chained-bridge-bandwidth", - "plugins": [ - { - "type": "bridge", - "ipam": { - "type": "host-local", - "subnet": "10.11.2.0/24" - } - }, - { - "type": "bandwidth", - "ingressRate": 800, - "ingressBurst": 200, - "egressRate": 800, - "egressBurst": 200 - } - ] -} diff --git a/integration/testdata/chained-bridge-bandwidth/network-chain-test.conflist b/integration/testdata/chained-bridge-bandwidth/network-chain-test.conflist new file mode 100644 index 000000000000..81a99b4d9d20 --- /dev/null +++ b/integration/testdata/chained-bridge-bandwidth/network-chain-test.conflist @@ -0,0 +1,23 @@ +{ + "cniVersion": "0.3.1", + "name": "network-chain-test", + "plugins": [ + { + "type": "bridge", + "bridge": "test-bridge-0", + "isDefaultGateway": true, + "ipam": { + "type": "host-local", + "subnet": "10.11.2.0/24", + "dataDir": "/tmp/foo" + } + }, + { + "type": "bandwidth", + "ingressRate": 8000, + "ingressBurst": 16000, + "egressRate": 8000, + "egressBurst": 16000 + } + ] +} diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go index 9835eef3f63c..ba09c5d3a40f 100644 --- a/plugins/meta/bandwidth/main.go +++ b/plugins/meta/bandwidth/main.go @@ -17,7 +17,6 @@ package main import ( "crypto/sha1" "encoding/json" - "errors" "fmt" "github.com/containernetworking/cni/pkg/skel" @@ -25,7 +24,9 @@ import ( "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" + "github.com/containernetworking/plugins/pkg/ip" "github.com/vishvananda/netlink" + "errors" ) type PluginConf struct { @@ -112,15 +113,16 @@ func getMTU(deviceName string) (int, error) { } func getHostInterface(interfaces []*current.Interface) (*current.Interface, error) { - for _, prevIface := range interfaces { - if prevIface.Sandbox != "" { - continue + var err error + for _, iface := range interfaces { + if iface.Sandbox == "" { // host interface + _, _, err = ip.GetVethPeerIfindex(iface.Name) + if err == nil { + return iface, err + } } - - return prevIface, nil } - - return nil, errors.New("no host interface found") + return nil, errors.New("no host interface found: " + err.Error()) } func cmdAdd(args *skel.CmdArgs) error { From fe0cf201f887a71b73efbd8b7d7ccc4e2e4066cc Mon Sep 17 00:00:00 2001 From: Tyler Schultz Date: Mon, 12 Mar 2018 15:53:23 -0700 Subject: [PATCH 8/8] Safely print error Format plugin code Signed-off-by: Aidan Obley --- integration/integration_linux_test.go | 50 +++++++++++---------------- plugins/meta/bandwidth/main.go | 8 +++-- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/integration/integration_linux_test.go b/integration/integration_linux_test.go index e593f76cb3b2..9c19193c2385 100644 --- a/integration/integration_linux_test.go +++ b/integration/integration_linux_test.go @@ -105,9 +105,6 @@ var _ = Describe("Basic PTP using cnitool", func() { basicBridgeEnv TestEnv chainedBridgeBandwidthEnv TestEnv chainedBridgeBandwidthSession, basicBridgeSession *gexec.Session - chainedBridgeBandwidthPort, basicBridgePort int - chainedBridgeIP, basicBridgeIP string - runtimeWithLimit, runtimeWithoutLimit time.Duration ) BeforeEach(func() { @@ -154,43 +151,38 @@ var _ = Describe("Basic PTP using cnitool", func() { Measure("limits traffic only on the restricted bandwith veth device", func(b Benchmarker) { ipRegexp := regexp.MustCompile("10\\.11\\.2\\.\\d{1,3}") - By(fmt.Sprintf("adding %s to %s\n\n", "chained-bridge-bandwidth", contNS1.ShortName()), func() { - chainedBridgeBandwidthEnv.runInNS(hostNS, cnitoolBin, "add", "network-chain-test", contNS1.LongName()) - chainedBridgeIP = ipRegexp.FindString(chainedBridgeBandwidthEnv.runInNS(contNS1, "ip", "addr")) - Expect(chainedBridgeIP).To(ContainSubstring("10.11.2.")) - }) + By(fmt.Sprintf("adding %s to %s\n\n", "chained-bridge-bandwidth", contNS1.ShortName())) + chainedBridgeBandwidthEnv.runInNS(hostNS, cnitoolBin, "add", "network-chain-test", contNS1.LongName()) + chainedBridgeIP := ipRegexp.FindString(chainedBridgeBandwidthEnv.runInNS(contNS1, "ip", "addr")) + Expect(chainedBridgeIP).To(ContainSubstring("10.11.2.")) - By(fmt.Sprintf("adding %s to %s\n\n", "basic-bridge", contNS2.ShortName()), func() { - basicBridgeEnv.runInNS(hostNS, cnitoolBin, "add", "network-chain-test", contNS2.LongName()) - basicBridgeIP = ipRegexp.FindString(basicBridgeEnv.runInNS(contNS2, "ip", "addr")) - Expect(basicBridgeIP).To(ContainSubstring("10.11.2.")) - }) + By(fmt.Sprintf("adding %s to %s\n\n", "basic-bridge", contNS2.ShortName())) + basicBridgeEnv.runInNS(hostNS, cnitoolBin, "add", "network-chain-test", contNS2.LongName()) + basicBridgeIP := ipRegexp.FindString(basicBridgeEnv.runInNS(contNS2, "ip", "addr")) + Expect(basicBridgeIP).To(ContainSubstring("10.11.2.")) + var chainedBridgeBandwidthPort, basicBridgePort int var err error - By(fmt.Sprintf("starting echo server in %s\n\n", contNS1.ShortName()), func() { - chainedBridgeBandwidthPort, chainedBridgeBandwidthSession, err = startEchoServerInNamespace(contNS1) - Expect(err).ToNot(HaveOccurred()) - }) + By(fmt.Sprintf("starting echo server in %s\n\n", contNS1.ShortName())) + chainedBridgeBandwidthPort, chainedBridgeBandwidthSession, err = startEchoServerInNamespace(contNS1) + Expect(err).ToNot(HaveOccurred()) - By(fmt.Sprintf("starting echo server in %s\n\n", contNS2.ShortName()), func() { - basicBridgePort, basicBridgeSession, err = startEchoServerInNamespace(contNS2) - Expect(err).ToNot(HaveOccurred()) - }) + By(fmt.Sprintf("starting echo server in %s\n\n", contNS2.ShortName())) + basicBridgePort, basicBridgeSession, err = startEchoServerInNamespace(contNS2) + Expect(err).ToNot(HaveOccurred()) packetInBytes := 20000 // The shaper needs to 'warm'. Send enough to cause it to throttle, // balanced by run time. - By(fmt.Sprintf("sending tcp traffic to the chained, bridged, traffic shaped container on ip address '%s:%d'\n\n", chainedBridgeIP, chainedBridgeBandwidthPort), func() { - runtimeWithLimit = b.Time("with chained bridge and bandwidth plugins", func() { - makeTcpClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes) - }) + By(fmt.Sprintf("sending tcp traffic to the chained, bridged, traffic shaped container on ip address '%s:%d'\n\n", chainedBridgeIP, chainedBridgeBandwidthPort)) + runtimeWithLimit := b.Time("with chained bridge and bandwidth plugins", func() { + makeTcpClientInNS(hostNS.ShortName(), chainedBridgeIP, chainedBridgeBandwidthPort, packetInBytes) }) - By(fmt.Sprintf("sending tcp traffic to the basic bridged container on ip address '%s:%d'\n\n", basicBridgeIP, basicBridgePort), func() { - runtimeWithoutLimit = b.Time("with basic bridged plugin", func() { - makeTcpClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes) - }) + By(fmt.Sprintf("sending tcp traffic to the basic bridged container on ip address '%s:%d'\n\n", basicBridgeIP, basicBridgePort)) + runtimeWithoutLimit := b.Time("with basic bridged plugin", func() { + makeTcpClientInNS(hostNS.ShortName(), basicBridgeIP, basicBridgePort, packetInBytes) }) Expect(runtimeWithLimit).To(BeNumerically(">", runtimeWithoutLimit+1000*time.Millisecond)) diff --git a/plugins/meta/bandwidth/main.go b/plugins/meta/bandwidth/main.go index ba09c5d3a40f..7aa7c39d9381 100644 --- a/plugins/meta/bandwidth/main.go +++ b/plugins/meta/bandwidth/main.go @@ -24,9 +24,9 @@ import ( "github.com/containernetworking/cni/pkg/types/current" "github.com/containernetworking/cni/pkg/version" + "errors" "github.com/containernetworking/plugins/pkg/ip" "github.com/vishvananda/netlink" - "errors" ) type PluginConf struct { @@ -113,6 +113,10 @@ func getMTU(deviceName string) (int, error) { } func getHostInterface(interfaces []*current.Interface) (*current.Interface, error) { + if len(interfaces) == 0 { + return nil, errors.New("no interfaces provided") + } + var err error for _, iface := range interfaces { if iface.Sandbox == "" { // host interface @@ -122,7 +126,7 @@ func getHostInterface(interfaces []*current.Interface) (*current.Interface, erro } } } - return nil, errors.New("no host interface found: " + err.Error()) + return nil, errors.New(fmt.Sprintf("no host interface found. last error: %s", err)) } func cmdAdd(args *skel.CmdArgs) error {