Skip to content

Commit

Permalink
masquerade packets from Pods to service IP
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Apr 20, 2022
1 parent 4faa883 commit 302156b
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 30 deletions.
2 changes: 2 additions & 0 deletions dist/images/uninstall.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ fi

iptables -t nat -D POSTROUTING -m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN
iptables -t nat -D POSTROUTING -m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE
iptables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000
iptables -t filter -D INPUT -m set --match-set ovn40subnets dst -j ACCEPT
iptables -t filter -D INPUT -m set --match-set ovn40subnets src -j ACCEPT
iptables -t filter -D INPUT -m set --match-set ovn40services dst -j ACCEPT
Expand All @@ -48,6 +49,7 @@ ipset destroy ovn40services

ip6tables -t nat -D POSTROUTING -m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN
ip6tables -t nat -D POSTROUTING -m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE
ip6tables -t mangle -D PREROUTING -i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000
ip6tables -t filter -D INPUT -m set --match-set ovn60subnets dst -j ACCEPT
ip6tables -t filter -D INPUT -m set --match-set ovn60subnets src -j ACCEPT
ip6tables -t filter -D INPUT -m set --match-set ovn60services dst -j ACCEPT
Expand Down
4 changes: 4 additions & 0 deletions pkg/daemon/gateway.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,8 @@ func (c *Controller) setIptables() error {
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn40subnets src -m set ! --match-set ovn40other-node src -m set --match-set ovn40subnets-nat dst -j RETURN`)},
// nat outgoing
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn40subnets-nat src -m set ! --match-set ovn40subnets dst -j MASQUERADE`)},
// mark packets from pod to service
{Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn40subnets src -m set --match-set ovn40services dst -j MARK --set-xmark 0x4000/0x4000`)},
// Input Accept
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets src -j ACCEPT`)},
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn40subnets dst -j ACCEPT`)},
Expand All @@ -404,6 +406,8 @@ func (c *Controller) setIptables() error {
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set ! --match-set ovn60subnets src -m set ! --match-set ovn60other-node src -m set --match-set ovn60subnets-nat dst -j RETURN`)},
// nat outgoing
{Table: "nat", Chain: "POSTROUTING", Rule: strings.Fields(`-m set --match-set ovn60subnets-nat src -m set ! --match-set ovn60subnets dst -j MASQUERADE`)},
// mark packets from pod to service
{Table: "mangle", Chain: "PREROUTING", Rule: strings.Fields(`-i ovn0 -m set --match-set ovn60subnets src -m set --match-set ovn60services dst -j MARK --set-xmark 0x4000/0x4000`)},
// Input Accept
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets src -j ACCEPT`)},
{Table: "filter", Chain: "INPUT", Rule: strings.Fields(`-m set --match-set ovn60subnets dst -j ACCEPT`)},
Expand Down
63 changes: 33 additions & 30 deletions test/e2e/service/service.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,20 @@ func hasEndpoint(node string, endpoints *corev1.Endpoints) bool {
return false
}

func checkService(shouldSucceed bool, cmd string, args ...string) {
c := exec.Command(cmd, args...)
var stdout, stderr bytes.Buffer
c.Stdout, c.Stderr = &stdout, &stderr
err := c.Run()
output := strings.TrimSpace(stdout.String())
if shouldSucceed {
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("stdout: %s, stderr: %s", output, strings.TrimSpace(stderr.String())))
Expect(output).To(Equal("200"))
} else {
Expect(err).To(HaveOccurred())
Expect(output).To(Equal("000"))
func checkService(checkCount int, shouldSucceed bool, cmd string, args ...string) {
for i := 0; i < checkCount; i++ {
c := exec.Command(cmd, args...)
var stdout, stderr bytes.Buffer
c.Stdout, c.Stderr = &stdout, &stderr
err := c.Run()
output := strings.TrimSpace(stdout.String())
if shouldSucceed {
Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("stdout: %s, stderr: %s", output, strings.TrimSpace(stderr.String())))
Expect(output).To(Equal("200"))
} else {
Expect(err).To(HaveOccurred())
Expect(output).To(Equal("000"))
}
}
}

Expand All @@ -125,6 +127,7 @@ var _ = Describe("[Service]", func() {

nodes, err := f.KubeClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
Expect(err).NotTo(HaveOccurred())
checkCount := len(nodes.Items)

var ciliumChaining, proxyIpvsMode bool
_, err = f.KubeClientSet.AppsV1().DaemonSets(namespace).Get(context.Background(), "cilium", metav1.GetOptions{})
Expand All @@ -148,7 +151,7 @@ var _ = Describe("[Service]", func() {
port := hostService.Spec.Ports[0].Port
for _, ip := range hostService.Spec.ClusterIPs {
for _, pod := range containerPods.Items {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
}
}
})
Expand All @@ -157,15 +160,15 @@ var _ = Describe("[Service]", func() {
port := hostService.Spec.Ports[0].Port
for _, ip := range hostService.Spec.ClusterIPs {
for _, pod := range hostPods.Items {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
}
}
})

It("external to ClusterIP", func() {
port := hostService.Spec.Ports[0].Port
for _, ip := range hostService.Spec.ClusterIPs {
checkService(true, "docker", append(dockerArgs, strings.Fields(curlArgs(ip, port))...)...)
checkService(checkCount, true, "docker", append(dockerArgs, strings.Fields(curlArgs(ip, port))...)...)
}
})

Expand All @@ -174,7 +177,7 @@ var _ = Describe("[Service]", func() {
for _, pod := range containerPods.Items {
for _, node := range nodes.Items {
for _, nodeIP := range nodeIPs(node) {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
}
}
Expand All @@ -185,7 +188,7 @@ var _ = Describe("[Service]", func() {
for _, pod := range hostPods.Items {
for _, node := range nodes.Items {
for _, nodeIP := range nodeIPs(node) {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
}
}
Expand All @@ -199,7 +202,7 @@ var _ = Describe("[Service]", func() {
port := hostService.Spec.Ports[0].NodePort
for _, node := range nodes.Items {
for _, nodeIP := range nodeIPs(node) {
checkService(true, "docker", append(dockerArgs, strings.Fields(curlArgs(nodeIP, port))...)...)
checkService(checkCount, true, "docker", append(dockerArgs, strings.Fields(curlArgs(nodeIP, port))...)...)
}
}
})
Expand All @@ -210,7 +213,7 @@ var _ = Describe("[Service]", func() {
port := containerService.Spec.Ports[0].Port
for _, ip := range containerService.Spec.ClusterIPs {
for _, pod := range containerPods.Items {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
}
}
})
Expand All @@ -219,15 +222,15 @@ var _ = Describe("[Service]", func() {
port := containerService.Spec.Ports[0].Port
for _, ip := range containerService.Spec.ClusterIPs {
for _, pod := range hostPods.Items {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
}
}
})

It("external to ClusterIP", func() {
port := containerService.Spec.Ports[0].Port
for _, ip := range containerService.Spec.ClusterIPs {
checkService(true, "docker", append(dockerArgs, strings.Fields(curlArgs(ip, port))...)...)
checkService(checkCount, true, "docker", append(dockerArgs, strings.Fields(curlArgs(ip, port))...)...)
}
})

Expand All @@ -236,7 +239,7 @@ var _ = Describe("[Service]", func() {
for _, pod := range containerPods.Items {
for _, node := range nodes.Items {
for _, nodeIP := range nodeIPs(node) {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
}
}
Expand All @@ -247,7 +250,7 @@ var _ = Describe("[Service]", func() {
for _, pod := range hostPods.Items {
for _, node := range nodes.Items {
for _, nodeIP := range nodeIPs(node) {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
}
}
Expand All @@ -261,7 +264,7 @@ var _ = Describe("[Service]", func() {
port := containerService.Spec.Ports[0].NodePort
for _, node := range nodes.Items {
for _, nodeIP := range nodeIPs(node) {
checkService(true, "docker", append(dockerArgs, strings.Fields(curlArgs(nodeIP, port))...)...)
checkService(checkCount, true, "docker", append(dockerArgs, strings.Fields(curlArgs(nodeIP, port))...)...)
}
}
})
Expand All @@ -272,7 +275,7 @@ var _ = Describe("[Service]", func() {
port := localEtpHostService.Spec.Ports[0].Port
for _, pod := range containerPods.Items {
for _, ip := range localEtpHostService.Spec.ClusterIPs {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
}
}
})
Expand All @@ -281,15 +284,15 @@ var _ = Describe("[Service]", func() {
port := localEtpHostService.Spec.Ports[0].Port
for _, pod := range hostPods.Items {
for _, ip := range localEtpHostService.Spec.ClusterIPs {
checkService(true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
checkService(checkCount, true, "kubectl", strings.Fields(kubectlArgs(pod.Name, ip, port))...)
}
}
})

It("external to ClusterIP", func() {
port := localEtpHostService.Spec.Ports[0].Port
for _, ip := range localEtpHostService.Spec.ClusterIPs {
checkService(true, "docker", append(dockerArgs, strings.Fields(curlArgs(ip, port))...)...)
checkService(checkCount, true, "docker", append(dockerArgs, strings.Fields(curlArgs(ip, port))...)...)
}
})

Expand All @@ -304,7 +307,7 @@ var _ = Describe("[Service]", func() {
for _, pod := range containerPods.Items {
shoudSucceed := hasEndpoint || !proxyIpvsMode
for _, nodeIP := range nodeIPs(node) {
checkService(shoudSucceed, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
checkService(checkCount, shoudSucceed, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
}
}
Expand All @@ -321,7 +324,7 @@ var _ = Describe("[Service]", func() {
for _, pod := range hostPods.Items {
shoudSucceed := hasEndpoint || (!proxyIpvsMode && pod.Spec.NodeName == node.Name)
for _, nodeIP := range nodeIPs(node) {
checkService(shoudSucceed, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
checkService(checkCount, shoudSucceed, "kubectl", strings.Fields(kubectlArgs(pod.Name, nodeIP, port))...)
}
}
}
Expand All @@ -336,7 +339,7 @@ var _ = Describe("[Service]", func() {
for _, node := range nodes.Items {
shouldSucceed := hasEndpoint(node.Name, localEtpHostEndpoints)
for _, nodeIP := range nodeIPs(node) {
checkService(shouldSucceed, "docker", append(dockerArgs, strings.Fields(curlArgs(nodeIP, port))...)...)
checkService(checkCount, shouldSucceed, "docker", append(dockerArgs, strings.Fields(curlArgs(nodeIP, port))...)...)
}
}
})
Expand Down

0 comments on commit 302156b

Please sign in to comment.