Skip to content

Commit

Permalink
Add IPsec e2e tests for node traffic
Browse files Browse the repository at this point in the history 
This commit adds and updates relevant tests for validating node traffic
for both east west and north south IPsec configurations.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>
  • Loading branch information
@pperiyasamy
pperiyasamy committed Apr 25, 2024
1 parent 9cdfe80 commit 3c971f2
Show file tree
Hide file tree
Showing 2 changed files with 96 additions and 40 deletions.
126 changes: 90 additions & 36 deletions test/extended/networking/ipsec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,9 @@ const (
// tcpdumpESPFilter can be used to filter out IPsec packets destined to target node.
tcpdumpESPFilter = "esp and src %s and dst %s"
// tcpdumpGeneveFilter can be used to filter out Geneve encapsulated packets destined to target node.
tcpdumpGeneveFilter = "udp port 6081 and src %s and dst %s"
tcpdumpGeneveFilter = "udp port 6081 and src %s and dst %s"
// tcpdumpICMPFilter can be used to filter out icmp packets destined to target node.
tcpdumpICMPFilter = "icmp and src %s and dst %s"
masterIPsecMachineConfigName = "80-ipsec-master-extensions"
workerIPSecMachineConfigName = "80-ipsec-worker-extensions"
ipsecRolloutWaitDuration = 20 * time.Minute
Expand Down Expand Up @@ -86,6 +88,14 @@ var (
certExpirationDate = time.Date(2034, time.April, 10, 0, 0, 0, 0, time.UTC)
)

type trafficType string

const (
esp trafficType = "esp"
geneve trafficType = "geneve"
icmp trafficType = "icmp"
)

// configureIPsecMode helps to rollout specified IPsec Mode on the cluster. If the cluster is already
// configured with specified mode, then this is almost like no-op for the cluster.
func configureIPsecMode(oc *exutil.CLI, ipsecMode v1.IPsecMode) error {
Expand Down Expand Up @@ -355,15 +365,22 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
var config *testConfig
// This function helps to generate ping traffic from src pod to dst pod and at the same time captures its
// node traffic on both src and dst node.
pingAndCheckNodeTraffic := func(src, dst *testNodeConfig, ipsecTraffic bool) error {
pingAndCheckNodeTraffic := func(src, dst *testNodeConfig, traffic trafficType) error {
tcpDumpSync := errgroup.Group{}
pingSync := errgroup.Group{}
var srcNodeTrafficFilter string
var dstNodeTrafficFilter string
// use tcpdump pod's ip address it's a node ip address because it's a hostnetworked pod.
srcNodeTrafficFilter := fmt.Sprintf(tcpdumpGeneveFilter, src.tcpdumpPod.Status.PodIP, dst.tcpdumpPod.Status.PodIP)
dstNodeTrafficFilter := fmt.Sprintf(tcpdumpGeneveFilter, dst.tcpdumpPod.Status.PodIP, src.tcpdumpPod.Status.PodIP)
if ipsecTraffic {
switch traffic {
case esp:
srcNodeTrafficFilter = fmt.Sprintf(tcpdumpESPFilter, src.tcpdumpPod.Status.PodIP, dst.tcpdumpPod.Status.PodIP)
dstNodeTrafficFilter = fmt.Sprintf(tcpdumpESPFilter, dst.tcpdumpPod.Status.PodIP, src.tcpdumpPod.Status.PodIP)
case geneve:
srcNodeTrafficFilter = fmt.Sprintf(tcpdumpGeneveFilter, src.tcpdumpPod.Status.PodIP, dst.tcpdumpPod.Status.PodIP)
dstNodeTrafficFilter = fmt.Sprintf(tcpdumpGeneveFilter, dst.tcpdumpPod.Status.PodIP, src.tcpdumpPod.Status.PodIP)
case icmp:
srcNodeTrafficFilter = fmt.Sprintf(tcpdumpICMPFilter, src.tcpdumpPod.Status.PodIP, dst.tcpdumpPod.Status.PodIP)
dstNodeTrafficFilter = fmt.Sprintf(tcpdumpICMPFilter, dst.tcpdumpPod.Status.PodIP, src.tcpdumpPod.Status.PodIP)
}
checkSrcNodeTraffic := func(src *testNodeConfig) error {
_, err := oc.AsAdmin().Run("exec").Args(src.tcpdumpPod.Name, "-n", src.tcpdumpPod.Namespace, "--",
Expand Down Expand Up @@ -406,7 +423,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
return nil
}

setupTestPods := func(config *testConfig) error {
setupTestPods := func(config *testConfig, isHostNetwork bool) error {
tcpdumpImage, err := exutil.DetermineImageFromRelease(oc, "network-tools")
o.Expect(err).NotTo(o.HaveOccurred())
createSync := errgroup.Group{}
Expand All @@ -418,6 +435,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
}
config.srcNodeConfig.pingPod = e2epod.CreateExecPodOrFail(context.TODO(), f.ClientSet, f.Namespace.Name, "ipsec-test-srcpod-", func(p *corev1.Pod) {
p.Spec.NodeName = config.srcNodeConfig.nodeName
p.Spec.HostNetwork = isHostNetwork
})
return err
})
Expand All @@ -429,6 +447,7 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
}
config.dstNodeConfig.pingPod = e2epod.CreateExecPodOrFail(context.TODO(), f.ClientSet, f.Namespace.Name, "ipsec-test-dstpod-", func(p *corev1.Pod) {
p.Spec.NodeName = config.dstNodeConfig.nodeName
p.Spec.HostNetwork = isHostNetwork
})
return err
})
Expand All @@ -451,8 +470,8 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
config.dstNodeConfig.tcpdumpPod = nil
}

checkForGeneveOnlyTraffic := func(config *testConfig) {
err := setupTestPods(config)
checkForGeneveOnlyPodTraffic := func(config *testConfig) {
err := setupTestPods(config, false)
o.Expect(err).NotTo(o.HaveOccurred())
defer func() {
// Don't cleanup test pods in error scenario.
Expand All @@ -461,14 +480,16 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
}
cleanupTestPods(config)
}()
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, false)
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, geneve)
o.Expect(err).NotTo(o.HaveOccurred())
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, true)
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, esp)
o.Expect(err).To(o.HaveOccurred())
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp)
o.Expect(err).To(o.HaveOccurred())
}

checkForESPOnlyTraffic := func(config *testConfig) {
err := setupTestPods(config)
checkForESPOnlyPodTraffic := func(config *testConfig) {
err := setupTestPods(config, false)
o.Expect(err).NotTo(o.HaveOccurred())
defer func() {
// Don't cleanup test pods in error scenario.
Expand All @@ -477,20 +498,47 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
}
cleanupTestPods(config)
}()
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, true)
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, esp)
o.Expect(err).NotTo(o.HaveOccurred())
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, false)
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, geneve)
o.Expect(err).To(o.HaveOccurred())
}

checkTraffic := func(mode v1.IPsecMode) {
checkPodTraffic := func(mode v1.IPsecMode) {
if mode == v1.IPsecModeFull {
checkForESPOnlyTraffic(config)
checkForESPOnlyPodTraffic(config)
} else {
checkForGeneveOnlyTraffic(config)
checkForGeneveOnlyPodTraffic(config)
}
}

checkNodeTraffic := func(ipsecForNode bool) {
err := setupTestPods(config, true)
o.Expect(err).NotTo(o.HaveOccurred())
defer func() {
// Don't cleanup test pods in error scenario.
if err != nil && !framework.TestContext.DeleteNamespaceOnFailure {
return
}
cleanupTestPods(config)
}()
if ipsecForNode {
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, esp)
o.Expect(err).NotTo(o.HaveOccurred())
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp)
o.Expect(err).To(o.HaveOccurred())
return
} else {
err = pingAndCheckNodeTraffic(config.srcNodeConfig, config.dstNodeConfig, icmp)
o.Expect(err).NotTo(o.HaveOccurred())
}
}

checkTraffic := func(mode v1.IPsecMode, ipsecForNode bool) {
checkPodTraffic(mode)
checkNodeTraffic(ipsecForNode)
}

g.BeforeAll(func() {
// Set up the config object with existing IPsecConfig, setup testing config on
// the selected nodes.
Expand Down Expand Up @@ -558,40 +606,40 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
}, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue())
})

g.DescribeTable("check traffic between local pod to a remote pod [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) {
g.DescribeTable("check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) {
o.Expect(config).NotTo(o.BeNil())
g.By("validate pod traffic before changing IPsec configuration")
// Ensure pod traffic is working with right encapsulation before rolling out IPsec configuration.
checkTraffic(config.ipsecMode)
g.By(fmt.Sprintf("configure IPsec in %s mode and validate pod traffic", mode))
g.By("validate traffic before changing IPsec configuration")
// Ensure pod traffic is working with right encapsulation and node specific traffic is unencrypted before rolling out IPsec configuration
checkTraffic(config.ipsecMode, false)
g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode))
err := configureIPsecMode(oc, mode)
o.Expect(err).NotTo(o.HaveOccurred())
waitForIPsecConfigToComplete(oc, mode)
// Ensure pod traffic is working with right encapsulation after rolling out IPsec configuration.
checkTraffic(mode)
// Also ensure node specific traffic is still unencrypted.
checkTraffic(mode, false)
},
g.Entry("with IPsec in full mode", v1.IPsecModeFull),
g.Entry("with IPsec in external mode", v1.IPsecModeExternal),
g.Entry("with IPsec in disabled mode", v1.IPsecModeDisabled),
)

// This test checks pod traffic to verify that N/S ipsec is enabled, and this wouldn't work to verify
// a working N/S ipsec in Full ipsec mode as in that case pod traffic would be encrypted anyway
// due to E/W ipsec configuration.
g.It("validate node traffic is IPsec encrypted for corresponding IPsec north south configuration [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func() {
g.DescribeTable("check traffic for north south IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) {
o.Expect(config).NotTo(o.BeNil())

g.By("validate pod traffic before changing IPsec configuration")
// Ensure pod traffic is working before rolling out IPsec configuration.
checkTraffic(config.ipsecMode)
g.By("validate traffic before changing IPsec configuration")
// Ensure pod traffic is working with right encapsulation and node specific traffic is unencrypted before rolling out IPsec configuration
checkTraffic(config.ipsecMode, false)

g.By("configure IPsec in External mode")
g.By(fmt.Sprintf("configure IPsec in %s mode and validate traffic", mode))
// Change IPsec mode to External and packet capture on the node's interface
// must be geneve encapsulated ones.
err := configureIPsecMode(oc, v1.IPsecModeExternal)
err := configureIPsecMode(oc, mode)
o.Expect(err).NotTo(o.HaveOccurred())
waitForIPsecConfigToComplete(oc, v1.IPsecModeExternal)
checkForGeneveOnlyTraffic(config)
waitForIPsecConfigToComplete(oc, mode)
// Ensure pod traffic is working with right encapsulation after rolling out IPsec configuration.
// Also ensure node specific traffic is still unencrypted.
checkTraffic(mode, false)

g.By("configure IPsec certs on the worker nodes")
// The certificates in the Machine Config has validity period of 120 months starting from April 11, 2024.
Expand Down Expand Up @@ -635,8 +683,14 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
waitForIPsecNSConfigApplied()

g.By("validate IPsec traffic between nodes")
checkForESPOnlyTraffic(config)
})
// Ensure pod traffic between nodes are encrypted.
checkForESPOnlyPodTraffic(config)
// Ensure node traffic is also encrypted.
checkNodeTraffic(true)
},
g.Entry("with IPsec in full mode", v1.IPsecModeFull),
g.Entry("with IPsec in external mode", v1.IPsecModeExternal),
)
})
})

Expand Down
10 changes: 6 additions & 4 deletions test/extended/util/annotate/generated/zz_generated.annotations.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 3c971f2

Please sign in to comment.