From 2421eb63175b54e326b5d5bbff8a14f49b05be25 Mon Sep 17 00:00:00 2001 From: Chun Chen Date: Tue, 22 Oct 2019 13:05:24 +0800 Subject: [PATCH] Reduce duplicate code --- e2e/helper/util.go | 8 ++++ e2e/k8s-vlan/bridge_test.go | 35 ++++++++--------- e2e/k8s-vlan/vlan_test.go | 37 +++++------------- pkg/network/portmapping/iptables.go | 59 +++++++++++------------------ pkg/policy/policy_test.go | 43 ++++++++++----------- 5 files changed, 74 insertions(+), 108 deletions(-) diff --git a/e2e/helper/util.go b/e2e/helper/util.go index 7f5c9605..7e1b032d 100644 --- a/e2e/helper/util.go +++ b/e2e/helper/util.go @@ -179,3 +179,11 @@ func Curl(ip, port string) ([]byte, error) { func CleanupCNIData(networkName string) error { return os.RemoveAll(path.Join("/var/lib/cni/networks/", networkName)) } + +func NewDummyVlan(cidrIPNet *net.IPNet, vlanId int) *LinkDevice { + return NewLinkDevice(nil, fmt.Sprintf("dummy0.%d", vlanId), "vlan").SetMaster( + NewLinkDevice(nil, fmt.Sprintf("br%d", vlanId), "bridge"), + ).SetParent( + NewLinkDevice(cidrIPNet, "dummy0", "dummy"), + ) +} \ No newline at end of file diff --git a/e2e/k8s-vlan/bridge_test.go b/e2e/k8s-vlan/bridge_test.go index 16b1bf15..1c0d4db8 100644 --- a/e2e/k8s-vlan/bridge_test.go +++ b/e2e/k8s-vlan/bridge_test.go @@ -13,6 +13,17 @@ var _ = Describe("galaxy-k8s-vlan bridge and pure test", func() { ifaceCidr := "192.168.0.66/26" containerCidr := "192.168.0.68/26" containerId := helper.NewContainerId() + cidrIPNet, _ := ips.ParseCIDR(ifaceCidr) + argsStr, _ := helper.IPInfo(containerCidr, 0) + + var cmdAdd = func(netConf []byte) string { + nsPath := helper.CmdAdd(containerId, ifaceCidr, argsStr, cni, + `{"cniVersion":"0.2.0","ip4":{"ip":"192.168.0.68/26","gateway":"192.168.0.65","routes":[{"dst":"0.0.0.0/0"}]},"dns":{}}`, netConf) + _, err := helper.Ping("192.168.0.68") + Expect(err).NotTo(HaveOccurred()) + return nsPath + } + AfterEach(func() { helper.CleanupNetNS() helper.CleanupDummy() @@ -25,17 +36,9 @@ var _ = Describe("galaxy-k8s-vlan bridge and pure test", func() { "device": "dummy0", "default_bridge_name": "brtest" }`) - argsStr, err := helper.IPInfo(containerCidr, 0) - Expect(err).NotTo(HaveOccurred()) - nsPath := helper.CmdAdd(containerId, ifaceCidr, argsStr, cni, - `{"cniVersion":"0.2.0","ip4":{"ip":"192.168.0.68/26","gateway":"192.168.0.65","routes":[{"dst":"0.0.0.0/0"}]},"dns":{}}`, netConf) - _, err = helper.Ping("192.168.0.68") - Expect(err).NotTo(HaveOccurred()) - + nsPath := cmdAdd(netConf) // check host iface topology, route, neigh, ip address is expected - cidrIPNet, err := ips.ParseCIDR(ifaceCidr) - Expect(err).NotTo(HaveOccurred()) - err = (&helper.NetworkTopology{ + err := (&helper.NetworkTopology{ LeaveDevices: []*helper.LinkDevice{ helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth").SetMaster( helper.NewLinkDevice(cidrIPNet, "brtest", "bridge"), @@ -61,17 +64,9 @@ var _ = Describe("galaxy-k8s-vlan bridge and pure test", func() { "device": "dummy0", "switch": "pure" }`) - argsStr, err := helper.IPInfo(containerCidr, 0) - Expect(err).NotTo(HaveOccurred()) - nsPath := helper.CmdAdd(containerId, ifaceCidr, argsStr, cni, - `{"cniVersion":"0.2.0","ip4":{"ip":"192.168.0.68/26","gateway":"192.168.0.65","routes":[{"dst":"0.0.0.0/0"}]},"dns":{}}`, netConf) - _, err = helper.Ping("192.168.0.68") - Expect(err).NotTo(HaveOccurred()) - + nsPath := cmdAdd(netConf) // check host iface topology, route, neigh, ip address is expected - cidrIPNet, err := ips.ParseCIDR(ifaceCidr) - Expect(err).NotTo(HaveOccurred()) - err = (&helper.NetworkTopology{ + err := (&helper.NetworkTopology{ LeaveDevices: []*helper.LinkDevice{ helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth"), helper.NewLinkDevice(cidrIPNet, "dummy0", "dummy"), diff --git a/e2e/k8s-vlan/vlan_test.go b/e2e/k8s-vlan/vlan_test.go index d7b98837..863f15eb 100644 --- a/e2e/k8s-vlan/vlan_test.go +++ b/e2e/k8s-vlan/vlan_test.go @@ -16,6 +16,11 @@ var _ = Describe("galaxy-k8s-vlan vlan test", func() { ifaceCidr := "192.168.0.66/26" containerCidr := "192.168.0.68/26" containerId := helper.NewContainerId() + cidrIPNet,_ := ips.ParseCIDR(ifaceCidr) + hostVeth1 := helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth").SetMaster( + helper.NewLinkDevice(nil, "br2", "bridge"), + ) + dummyVlan2 := helper.NewDummyVlan(cidrIPNet, 2) AfterEach(func() { helper.CleanupNetNS() helper.CleanupDummy() @@ -40,19 +45,8 @@ var _ = Describe("galaxy-k8s-vlan vlan test", func() { //Expect(err).To(HaveOccurred()) // vlan is not reachable on host // check host iface topology, route, neigh, ip address is expected - cidrIPNet, err := ips.ParseCIDR(ifaceCidr) - Expect(err).NotTo(HaveOccurred()) err = (&helper.NetworkTopology{ - LeaveDevices: []*helper.LinkDevice{ - helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth").SetMaster( - helper.NewLinkDevice(nil, "br2", "bridge"), - ), - helper.NewLinkDevice(nil, "dummy0.2", "vlan").SetMaster( - helper.NewLinkDevice(nil, "br2", "bridge"), - ).SetParent( - helper.NewLinkDevice(cidrIPNet, "dummy0", "dummy"), - ), - }, + LeaveDevices: []*helper.LinkDevice{hostVeth1,dummyVlan2}, }).Verify() Expect(err).Should(BeNil(), "%v", err) @@ -78,27 +72,14 @@ var _ = Describe("galaxy-k8s-vlan vlan test", func() { `{"cniVersion":"0.2.0","ip4":{"ip":"192.168.0.68/26","gateway":"192.168.0.65","routes":[{"dst":"10.0.0.0/8"},{"dst":"172.16.0.0/12"},{"dst":"192.168.0.0/16"}]},"dns":{}}`, netConf) // check host iface topology, route, neigh, ip address is expected - cidrIPNet, err := ips.ParseCIDR(ifaceCidr) - Expect(err).NotTo(HaveOccurred()) err = (&helper.NetworkTopology{ LeaveDevices: []*helper.LinkDevice{ - helper.NewLinkDevice(nil, utils.HostVethName(containerId, ""), "veth").SetMaster( - helper.NewLinkDevice(nil, "br2", "bridge"), - ), - helper.NewLinkDevice(nil, "dummy0.2", "vlan").SetMaster( - helper.NewLinkDevice(nil, "br2", "bridge"), - ).SetParent( - helper.NewLinkDevice(cidrIPNet, "dummy0", "dummy"), - ), - + hostVeth1, + dummyVlan2, helper.NewLinkDevice(nil, utils.HostVethName(containerId, "-2"), "veth").SetMaster( helper.NewLinkDevice(nil, "br3", "bridge"), ), - helper.NewLinkDevice(nil, "dummy0.3", "vlan").SetMaster( - helper.NewLinkDevice(nil, "br3", "bridge"), - ).SetParent( - helper.NewLinkDevice(cidrIPNet, "dummy0", "dummy"), - ), + helper.NewDummyVlan(cidrIPNet, 3), }, }).Verify() Expect(err).Should(BeNil(), "%v", err) diff --git a/pkg/network/portmapping/iptables.go b/pkg/network/portmapping/iptables.go index baa99a2f..3167ce8d 100644 --- a/pkg/network/portmapping/iptables.go +++ b/pkg/network/portmapping/iptables.go @@ -68,24 +68,7 @@ func (h *PortMappingHandler) SetupPortMapping(ports []k8s.Port) error { args = append(args, "-j", string(hostportChain)) kubeHostportsChainRules = append(kubeHostportsChainRules, args) - // Assuming kubelet is syncing iptables KUBE-MARK-MASQ chain - // If the request comes from the pod that is serving the hostport, then SNAT - args = []string{ - "-A", string(hostportChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, containerPort.PodName, containerPort.HostPort), - "-s", containerPort.PodIP, "-j", string(KubeMarkMasqChain), - } - writeLine(natRules, args...) - - // Create hostport chain to DNAT traffic to final destination - // IPTables will maintained the stats for this chain - args = []string{ - "-A", string(hostportChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, containerPort.PodName, containerPort.HostPort), - "-m", protocol, "-p", protocol, - "-j", "DNAT", fmt.Sprintf("--to-destination=%s:%d", containerPort.PodIP, containerPort.ContainerPort), - } - writeLine(natRules, args...) + containerPortChainRules(&containerPort, protocol, hostportChain, natRules) } writeLine(natRules, "COMMIT") @@ -104,6 +87,27 @@ func (h *PortMappingHandler) SetupPortMapping(ports []k8s.Port) error { return nil } +func containerPortChainRules(containerPort *k8s.Port, protocol string, hostportChain utiliptables.Chain, natRules *bytes.Buffer) { + // Assuming kubelet is syncing iptables KUBE-MARK-MASQ chain + // If the request comes from the pod that is serving the hostport, then SNAT + args := []string{ + "-A", string(hostportChain), + "-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, containerPort.PodName, containerPort.HostPort), + "-s", containerPort.PodIP, "-j", string(KubeMarkMasqChain), + } + writeLine(natRules, args...) + + // Create hostport chain to DNAT traffic to final destination + // IPTables will maintained the stats for this chain + args = []string{ + "-A", string(hostportChain), + "-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, containerPort.PodName, containerPort.HostPort), + "-m", protocol, "-p", protocol, + "-j", "DNAT", fmt.Sprintf("--to-destination=%s:%d", containerPort.PodIP, containerPort.ContainerPort), + } + writeLine(natRules, args...) +} + func (h *PortMappingHandler) CleanPortMapping(ports []k8s.Port) { var kubeHostportsChainRules [][]string natChains := bytes.NewBuffer(nil) @@ -198,24 +202,7 @@ func (h *PortMappingHandler) SetupPortMappingForAllPods(ports []k8s.Port) error } writeLine(natRules, args...) - // Assuming kubelet is syncing iptables KUBE-MARK-MASQ chain - // If the request comes from the pod that is serving the hostport, then SNAT - args = []string{ - "-A", string(hostportChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, containerPort.PodName, containerPort.HostPort), - "-s", containerPort.PodIP, "-j", string(KubeMarkMasqChain), - } - writeLine(natRules, args...) - - // Create hostport chain to DNAT traffic to final destination - // IPTables will maintained the stats for this chain - args = []string{ - "-A", string(hostportChain), - "-m", "comment", "--comment", fmt.Sprintf(`"%s hostport %d"`, containerPort.PodName, containerPort.HostPort), - "-m", protocol, "-p", protocol, - "-j", "DNAT", fmt.Sprintf("--to-destination=%s:%d", containerPort.PodIP, containerPort.ContainerPort), - } - writeLine(natRules, args...) + containerPortChainRules(&containerPort, protocol, hostportChain, natRules) } // Delete chains no longer in use. diff --git a/pkg/policy/policy_test.go b/pkg/policy/policy_test.go index 3c27842e..fe44c277 100644 --- a/pkg/policy/policy_test.go +++ b/pkg/policy/policy_test.go @@ -24,6 +24,21 @@ func newTestPolicyManager() *PolicyManager { } } +var ( + ipTable1 = &ipsetTable{ + IPSet: ipset.IPSet{Name: "GLX-sip-0-XX2", SetType: ipset.HashIP}, + entries: []ipset.Entry{ + {IP: "1.1.0.1", SetType: ipset.HashIP}, {IP: "1.1.0.2", SetType: ipset.HashIP}, + }, + } + natTable1 = &ipsetTable{ + IPSet: ipset.IPSet{Name: "GLX-snet-1-XX3", SetType: ipset.HashNet}, + entries: []ipset.Entry{ + {Net: "2.1.0.0/24", SetType: ipset.HashNet}, {Net: "2.1.0.2/32", SetType: ipset.HashNet, Options: []string{"nomatch"}}, {Net: "2.1.1.2/32", SetType: ipset.HashNet}, + }, + } +) + func TestSyncRules(t *testing.T) { pm := newTestPolicyManager() policies := []policy{ @@ -48,18 +63,8 @@ func TestSyncRules(t *testing.T) { ingressRule: &ingressRule{ srcRules: []rule{ { - ipTable: &ipsetTable{ - IPSet: ipset.IPSet{Name: "GLX-sip-0-XX2", SetType: ipset.HashIP}, - entries: []ipset.Entry{ - {IP: "1.1.0.1", SetType: ipset.HashIP}, {IP: "1.1.0.2", SetType: ipset.HashIP}, - }, - }, - netTable: &ipsetTable{ - IPSet: ipset.IPSet{Name: "GLX-snet-1-XX3", SetType: ipset.HashNet}, - entries: []ipset.Entry{ - {Net: "2.1.0.0/24", SetType: ipset.HashNet}, {Net: "2.1.0.2/32", SetType: ipset.HashNet, Options: []string{"nomatch"}}, {Net: "2.1.1.2/32", SetType: ipset.HashNet}, - }, - }, + ipTable: ipTable1, + netTable: natTable1, tcpPorts: []string{"53", "80"}, udpPorts: []string{"53", "80"}, }, @@ -189,18 +194,8 @@ func TestSyncPodChains(t *testing.T) { ingressRule: &ingressRule{ srcRules: []rule{ { - ipTable: &ipsetTable{ - IPSet: ipset.IPSet{Name: "GLX-sip-0-XX2", SetType: ipset.HashIP}, - entries: []ipset.Entry{ - {IP: "1.1.0.1", SetType: ipset.HashIP}, {IP: "1.1.0.2", SetType: ipset.HashIP}, - }, - }, - netTable: &ipsetTable{ - IPSet: ipset.IPSet{Name: "GLX-snet-1-XX3", SetType: ipset.HashNet}, - entries: []ipset.Entry{ - {Net: "2.1.0.0/24", SetType: ipset.HashNet}, {Net: "2.1.0.2/32", SetType: ipset.HashNet, Options: []string{"nomatch"}}, {Net: "2.1.1.2/32", SetType: ipset.HashNet}, - }, - }, + ipTable: ipTable1, + netTable: natTable1, tcpPorts: []string{"80"}, }, },