Skip to content

Commit

Permalink
Fix for issues from dev testing
Browse files Browse the repository at this point in the history 
This commit has fixes for issues found with ipsec tests while testing
with a 4.16 cluster running in AWS.

Signed-off-by: Periyasamy Palanisamy <pepalani@redhat.com>
  • Loading branch information
@pperiyasamy
pperiyasamy committed May 3, 2024
1 parent 83c3ed5 commit 125282d
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 99 deletions.
86 changes: 38 additions & 48 deletions test/extended/networking/ipsec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,7 @@ import (
v1 "github.com/openshift/api/operator/v1"
exutil "github.com/openshift/origin/test/extended/util"
"golang.org/x/sync/errgroup"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/retry"
Expand Down Expand Up @@ -42,6 +40,8 @@ const (
rightNodeIPsecPolicyName = "right-node-ipsec-policy"
leftNodeIPsecConfigYaml = "ipsec-left-node.yaml"
rightNodeIPsecConfigYaml = "ipsec-right-node.yaml"
ovnNamespace = "openshift-ovn-kubernetes"
ovnIPsecDsName = "ovn-ipsec-host"
)

// TODO: consider bringing in the NNCP api.
Expand Down Expand Up @@ -136,13 +136,16 @@ func getIPsecMode(oc *exutil.CLI) (v1.IPsecMode, error) {
// is completely ready on the cluster and cluster operators are coming back into ready state
// once ipsec rollout is complete.
func ensureIPsecEnabled(oc *exutil.CLI) error {
err := ensureIPsecMachineConfigRolloutComplete(oc)
if err != nil {
return err
}
return wait.PollUntilContextTimeout(context.Background(), ipsecRolloutWaitInterval,
ipsecRolloutWaitDuration, true, func(ctx context.Context) (bool, error) {
done, err := isIPsecDaemonSetRunning(oc)
done, err := areMachineConfigPoolsReadyWithIPsec(oc)
if err != nil && !isConnResetErr(err) {
return false, err
}
if !done {
return false, nil
}
done, err = isDaemonSetRunning(oc, ovnNamespace, ovnIPsecDsName)
if err != nil && !isConnResetErr(err) {
return false, err
}
Expand Down Expand Up @@ -182,11 +185,17 @@ func ensureIPsecMachineConfigRolloutComplete(oc *exutil.CLI) error {
func ensureIPsecDisabled(oc *exutil.CLI) error {
return wait.PollUntilContextTimeout(context.Background(), ipsecRolloutWaitInterval,
ipsecRolloutWaitDuration, true, func(ctx context.Context) (bool, error) {
ds, err := getIPsecDaemonSet(oc)
done, err := areMachineConfigPoolsReadyWithoutIPsec(oc)
if err != nil && !isConnResetErr(err) {
return false, err
}
if !done {
return false, nil
}
ds, err := getDaemonSet(oc, ovnNamespace, ovnIPsecDsName)
if err != nil && !isConnResetErr(err) {
return false, err
}
var done bool
if ds == nil && err == nil {
done, err = areClusterOperatorsReady((oc))
if err != nil && !isConnResetErr(err) {
Expand All @@ -197,24 +206,6 @@ func ensureIPsecDisabled(oc *exutil.CLI) error {
})
}

func isIPsecDaemonSetRunning(oc *exutil.CLI) (bool, error) {
ipsecDS, err := getIPsecDaemonSet(oc)
if ipsecDS == nil {
return false, err
}
// Be sure that it has ovn-ipsec-host pod running in each node.
ready := ipsecDS.Status.DesiredNumberScheduled == ipsecDS.Status.NumberReady
return ready, nil
}

func getIPsecDaemonSet(oc *exutil.CLI) (*appsv1.DaemonSet, error) {
ds, err := oc.AdminKubeClient().AppsV1().DaemonSets("openshift-ovn-kubernetes").Get(context.Background(), "ovn-ipsec-host", metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
return nil, nil
}
return ds, err
}

var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {

oc := exutil.NewCLIWithPodSecurityLevel("ipsec", admissionapi.LevelPrivileged)
Expand Down Expand Up @@ -325,10 +316,11 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
if err != nil {
return err
}
config.srcNodeConfig.pingPod = e2epod.CreateExecPodOrFail(context.TODO(), f.ClientSet, f.Namespace.Name, "ipsec-test-srcpod-", func(p *corev1.Pod) {
srcPingPod := e2epod.CreateExecPodOrFail(context.TODO(), f.ClientSet, f.Namespace.Name, "ipsec-test-srcpod-", func(p *corev1.Pod) {
p.Spec.NodeName = config.srcNodeConfig.nodeName
p.Spec.HostNetwork = isHostNetwork
})
config.srcNodeConfig.pingPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), srcPingPod.Name, metav1.GetOptions{})
return err
})
createSync.Go(func() error {
Expand All @@ -337,10 +329,11 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
if err != nil {
return err
}
config.dstNodeConfig.pingPod = e2epod.CreateExecPodOrFail(context.TODO(), f.ClientSet, f.Namespace.Name, "ipsec-test-dstpod-", func(p *corev1.Pod) {
dstPingPod := e2epod.CreateExecPodOrFail(context.TODO(), f.ClientSet, f.Namespace.Name, "ipsec-test-dstpod-", func(p *corev1.Pod) {
p.Spec.NodeName = config.dstNodeConfig.nodeName
p.Spec.HostNetwork = isHostNetwork
})
config.dstNodeConfig.pingPod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), dstPingPod.Name, metav1.GetOptions{})
return err
})
return createSync.Wait()
Expand Down Expand Up @@ -465,32 +458,29 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
})

g.AfterEach(func() {
// Restore the cluster back into original state after running all the tests.
g.By("restoring ipsec config into original state")
err := configureIPsecMode(oc, config.ipsecMode)
g.By("removing IPsec certs from worker nodes")
err := deleteNSCertMachineConfig(oc)
o.Expect(err).NotTo(o.HaveOccurred())
waitForIPsecConfigToComplete(oc, config.ipsecMode)
o.Eventually(func() bool {
pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel)
o.Expect(err).NotTo(o.HaveOccurred())
return areMachineConfigPoolsReadyWithoutMachineConfig(pools, nsCertMachineConfigName)
}, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue())

g.By("remove right node ipsec configuration")
err = oc.AsAdmin().Run("delete").Args("-f", rightNodeIPsecConfigYaml).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
oc.AsAdmin().Run("delete").Args("-f", rightNodeIPsecConfigYaml).Execute()

g.By("remove left node ipsec configuration")
err = oc.AsAdmin().Run("delete").Args("-f", leftNodeIPsecConfigYaml).Execute()
o.Expect(err).NotTo(o.HaveOccurred())
oc.AsAdmin().Run("delete").Args("-f", leftNodeIPsecConfigYaml).Execute()

g.By("undeploy nmstate handler")
err = undeployNmstateHandler(oc)
o.Expect(err).NotTo(o.HaveOccurred())
undeployNmstateHandler(oc)

g.By("removing IPsec certs from worker nodes")
err = deleteNSCertMachineConfig(oc)
// Restore the cluster back into original state after running all the tests.
g.By("restoring ipsec config into original state")
err = configureIPsecMode(oc, config.ipsecMode)
o.Expect(err).NotTo(o.HaveOccurred())
o.Eventually(func() bool {
ready, err := areWorkerMachineConfigPoolsReady(oc, nsCertMachineConfigName, false)
o.Expect(err).NotTo(o.HaveOccurred())
return ready
}, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue())
waitForIPsecConfigToComplete(oc, config.ipsecMode)
})

g.DescribeTable("check traffic for east west IPsec [apigroup:config.openshift.io] [Suite:openshift/network/ipsec]", func(mode v1.IPsecMode) {
Expand Down Expand Up @@ -540,9 +530,9 @@ var _ = g.Describe("[sig-network][Feature:IPsec]", g.Ordered, func() {
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(nsCertMachineConfig).NotTo(o.BeNil())
o.Eventually(func() bool {
exists, err := areWorkerMachineConfigPoolsReady(oc, nsCertMachineConfigName, true)
pools, err := getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel)
o.Expect(err).NotTo(o.HaveOccurred())
return exists
return areMachineConfigPoolsReadyWithMachineConfig(pools, nsCertMachineConfigName)
}, ipsecRolloutWaitDuration, ipsecRolloutWaitInterval).Should(o.BeTrue())

// Deploy nmstate handler which is used for rolling out IPsec config
Expand Down
121 changes: 70 additions & 51 deletions test/extended/networking/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,13 @@ import (
networkclient "github.com/openshift/client-go/network/clientset/versioned/typed/network/v1"
"github.com/openshift/library-go/pkg/network/networkutils"
exutil "github.com/openshift/origin/test/extended/util"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
kapierrs "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/wait"
Expand Down Expand Up @@ -73,6 +75,13 @@ const (
IPv6 IPFamily = "ipv6"
DualStack IPFamily = "dual"
Unknown IPFamily = "unknown"

nmstateNamespace = "openshift-nmstate"
)

var (
masterRoleMachineConfigLabel = map[string]string{"machineconfiguration.openshift.io/role": "master"}
workerRoleMachineConfigLabel = map[string]string{"machineconfiguration.openshift.io/role": "worker"}
)

// IsIPv6 returns true if a group of ips are ipv6.
Expand Down Expand Up @@ -698,24 +707,24 @@ func GetIPAddressFamily(oc *exutil.CLI) (bool, bool, error) {
}

func deployNmstateHandler(oc *exutil.CLI) error {
err := waitForDeploymentComplete(oc, "openshift-nmstate", "nmstate-operator")
err := waitForDeploymentComplete(oc, nmstateNamespace, "nmstate-operator")
if err != nil {
return fmt.Errorf("nmstate operator is not running: %v", err)
}
nmStateConfigYaml := exutil.FixturePath("testdata", "ipsec", nmstateConfigureManifestFile)
err = oc.AsAdmin().Run("create").Args("-f", nmStateConfigYaml, "--namespace=openshift-nmstate").Execute()
err = oc.AsAdmin().Run("create").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute()
if err != nil {
return fmt.Errorf("error configuring nmstate: %v", err)
}
err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true,
func(ctx context.Context) (bool, error) {
// Ensure nmstate handler is running.
return isDaemonSetRunning(oc, "openshift-nmstate", "nmstate-handler")
return isDaemonSetRunning(oc, nmstateNamespace, "nmstate-handler")
})
if err != nil {
return fmt.Errorf("failed to get nmstate handler running: %v", err)
}
err = waitForDeploymentComplete(oc, "openshift-nmstate", "nmstate-webhook")
err = waitForDeploymentComplete(oc, nmstateNamespace, "nmstate-webhook")
if err != nil {
return fmt.Errorf("nmstate webhook is not running: %v", err)
}
Expand All @@ -724,13 +733,13 @@ func deployNmstateHandler(oc *exutil.CLI) error {

func undeployNmstateHandler(oc *exutil.CLI) error {
nmStateConfigYaml := exutil.FixturePath("testdata", "ipsec", nmstateConfigureManifestFile)
err := oc.AsAdmin().Run("delete").Args("-f", nmStateConfigYaml, "--namespace=openshift-nmstate").Execute()
err := oc.AsAdmin().Run("delete").Args("-f", nmStateConfigYaml, fmt.Sprintf("--namespace=%s", nmstateNamespace)).Execute()
if err != nil {
return fmt.Errorf("error deleting nmstate configuration: %v", err)
}
err = wait.PollUntilContextTimeout(context.Background(), poll, 2*time.Minute, true,
func(ctx context.Context) (bool, error) {
_, err := oc.AdminKubeClient().AppsV1().DaemonSets("openshift-nmstate").Get(context.Background(), "nmstate-handler", metav1.GetOptions{})
_, err := oc.AdminKubeClient().AppsV1().DaemonSets(nmstateNamespace).Get(context.Background(), "nmstate-handler", metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
return true, nil
}
Expand All @@ -747,21 +756,30 @@ func waitForDeploymentComplete(oc *exutil.CLI, namespace, name string) error {
if err != nil {
return err
}
return e2edeployment.WaitForDeploymentComplete(oc.KubeClient(), deployment)
return e2edeployment.WaitForDeploymentComplete(oc.AdminKubeClient(), deployment)
}

func isDaemonSetRunning(oc *exutil.CLI, namespace, name string) (bool, error) {
ds, err := oc.AdminKubeClient().AppsV1().DaemonSets(namespace).Get(context.Background(), name, metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
return false, nil
} else if err != nil {
ds, err := getDaemonSet(oc, namespace, name)
if err != nil {
return false, err
}
if ds == nil {
return false, nil
}
// Be sure that it has ds pod running in each node.
desired, scheduled, ready := ds.Status.DesiredNumberScheduled, ds.Status.CurrentNumberScheduled, ds.Status.NumberReady
return desired == scheduled && desired == ready, nil
}

func getDaemonSet(oc *exutil.CLI, namespace, name string) (*appsv1.DaemonSet, error) {
ds, err := oc.AdminKubeClient().AppsV1().DaemonSets(namespace).Get(context.Background(), name, metav1.GetOptions{})
if err != nil && apierrors.IsNotFound(err) {
return nil, nil
}
return ds, err
}

func createIPsecCertsMachineConfig(oc *exutil.CLI) (*mcfgv1.MachineConfig, error) {
ipSecCertsMachineConfig := exutil.FixturePath("testdata", "ipsec", nsCertMachineConfigFile)
err := oc.AsAdmin().Run("create").Args("-f", ipSecCertsMachineConfig).Execute()
Expand Down Expand Up @@ -851,59 +869,38 @@ func isConnResetErr(err error) bool {
// This checks master and worker role machine config pools status are set with ipsec
// extension which confirms extension is successfully rolled out on all nodes.
func areMachineConfigPoolsReadyWithIPsec(oc *exutil.CLI) (bool, error) {
masterWithIPsec, err := areMasterMachineConfigPoolsWithIPsec(oc)
pools, err := getMachineConfigPoolByLabel(oc, masterRoleMachineConfigLabel)
if err != nil {
return false, err
}
workerWithIPsec, err := areWorkerMachineConfigPoolsReadyWithIPsec(oc)
masterWithIPsec := areMachineConfigPoolsReadyWithMachineConfig(pools, masterIPsecMachineConfigName)
pools, err = getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel)
if err != nil {
return false, err
}
return masterWithIPsec && workerWithIPsec, nil
}

func areMasterMachineConfigPoolsWithIPsec(oc *exutil.CLI) (bool, error) {
return areMachineConfigPoolsReady(oc, "machineconfiguration.openshift.io/role=master", masterIPsecMachineConfigName, true)
}

func areWorkerMachineConfigPoolsReadyWithIPsec(oc *exutil.CLI) (bool, error) {
return areWorkerMachineConfigPoolsReady(oc, workerIPSecMachineConfigName, true)
}
workerWithIPsec := areMachineConfigPoolsReadyWithMachineConfig(pools, workerIPSecMachineConfigName)

func areWorkerMachineConfigPoolsReady(oc *exutil.CLI, machineConfigName string, mustExist bool) (bool, error) {
return areMachineConfigPoolsReady(oc, "machineconfiguration.openshift.io/role=worker", machineConfigName, mustExist)
return masterWithIPsec && workerWithIPsec, nil
}

func areMachineConfigPoolsReady(oc *exutil.CLI, mcpSelectorLabel string, machineConfigName string, mustExist bool) (bool, error) {
poolList, err := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigPools().List(context.Background(),
metav1.ListOptions{})
// This checks master and worker role machine config pools status are set without ipsec
// extension which confirms extension is successfully removed from all nodes.
func areMachineConfigPoolsReadyWithoutIPsec(oc *exutil.CLI) (bool, error) {
pools, err := getMachineConfigPoolByLabel(oc, masterRoleMachineConfigLabel)
if err != nil {
return false, fmt.Errorf("failed to get ipsec machine config pools: %v", err)
}
if len(poolList.Items) == 0 {
return false, fmt.Errorf("no machine config pools found")
}
var pools []mcfgv1.MachineConfigPool
for _, pool := range poolList.Items {
if pool.Spec.MachineConfigSelector == nil {
continue
}
for lKey, lValue := range pool.Spec.MachineConfigSelector.MatchLabels {
if mcpSelectorLabel == fmt.Sprintf("%s=%s", lKey, lValue) {
pools = append(pools, pool)
}
}
}
if len(pools) == 0 {
return false, fmt.Errorf("empty machine config pools found for the selector")
return false, err
}
if mustExist {
return isMachineConfigReadyInPools(pools, machineConfigName), nil
masterWithoutIPsec := areMachineConfigPoolsReadyWithoutMachineConfig(pools, masterIPsecMachineConfigName)
pools, err = getMachineConfigPoolByLabel(oc, workerRoleMachineConfigLabel)
if err != nil {
return false, err
}
return areMachineConfigPoolReadyWithoutMachineConfig(pools, machineConfigName), nil
workerWithoutIPsec := areMachineConfigPoolsReadyWithoutMachineConfig(pools, workerIPSecMachineConfigName)

return masterWithoutIPsec && workerWithoutIPsec, nil
}

func isMachineConfigReadyInPools(pools []mcfgv1.MachineConfigPool, machineConfigName string) bool {
func areMachineConfigPoolsReadyWithMachineConfig(pools []mcfgv1.MachineConfigPool, machineConfigName string) bool {
mcExistsInPool := func(status mcfgv1.MachineConfigPoolStatus, name string) bool {
return status.MachineCount == status.UpdatedMachineCount &&
hasSourceInMachineConfigStatus(status, name)
Expand All @@ -916,7 +913,7 @@ func isMachineConfigReadyInPools(pools []mcfgv1.MachineConfigPool, machineConfig
return true
}

func areMachineConfigPoolReadyWithoutMachineConfig(pools []mcfgv1.MachineConfigPool, machineConfigName string) bool {
func areMachineConfigPoolsReadyWithoutMachineConfig(pools []mcfgv1.MachineConfigPool, machineConfigName string) bool {
mcNotExistsInPool := func(status mcfgv1.MachineConfigPoolStatus, name string) bool {
return status.MachineCount == status.UpdatedMachineCount &&
!hasSourceInMachineConfigStatus(status, name)
Expand Down Expand Up @@ -965,3 +962,25 @@ func areClusterOperatorsReady(oc *exutil.CLI) (bool, error) {
}
return true, nil
}

func getMachineConfigPoolByLabel(oc *exutil.CLI, mcSelectorLabel labels.Set) ([]mcfgv1.MachineConfigPool, error) {
poolList, err := oc.MachineConfigurationClient().MachineconfigurationV1().MachineConfigPools().List(context.Background(),
metav1.ListOptions{})
if err != nil {
return nil, err
}
var pools []mcfgv1.MachineConfigPool
for _, pool := range poolList.Items {
mcSelector, err := metav1.LabelSelectorAsSelector(pool.Spec.MachineConfigSelector)
if err != nil {
return nil, fmt.Errorf("invalid machine config label selector in %s pool", pool.Name)
}
if mcSelector.Matches(mcSelectorLabel) {
pools = append(pools, pool)
}
}
if len(pools) == 0 {
return nil, fmt.Errorf("empty machine config pools found for the selector")
}
return pools, nil
}

0 comments on commit 125282d

Please sign in to comment.