Merge pull request #28739 from jcho02/OCPBUGS-18534

OCPBUGS-18534: monitor test fix to wait before connecting to a non-existent dns on PowerVS and IBMCloud platforms

pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go

@@ -9,6 +9,7 @@ import (
 	"net/http"
 	"os"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/openshift/origin/pkg/monitortestframework"
@@ -203,6 +204,15 @@ func (w *availability) StartCollection(ctx context.Context, adminRESTConfig *res
 		return fmt.Errorf("error creating PDB: %w", err)
 	}
 
+	// On certain platforms hitting the hostname before it is ready leads to a blackhole, this code checks
+	// the host from the cluster's context
+	if infra.Spec.PlatformSpec.Type == configv1.PowerVSPlatformType || infra.Spec.PlatformSpec.Type == configv1.IBMCloudPlatformType {
+		nodeTgt := "node/" + nodeList.Items[0].ObjectMeta.Name
+		if err := checkHostnameReady(ctx, tcpService, nodeTgt); err != nil {
+			return err
+		}
+	}
+
 	// Hit it once before considering ourselves ready
 	fmt.Fprintf(os.Stderr, "hitting pods through the service's LoadBalancer\n")
 	timeout := 10 * time.Minute
@@ -348,3 +358,29 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re
 
 	return client.Get(url)
 }
+
+// Uses the first node in the cluster to verify the LoadBalancer host is active before returning
+func checkHostnameReady(ctx context.Context, tcpService *corev1.Service, nodeTgt string) error {
+	oc := exutil.NewCLIForMonitorTest(tcpService.GetObjectMeta().GetNamespace())
+
+	var (
+		stdOut string
+		err    error
+	)
+
+	wait.PollUntilContextTimeout(ctx, 15*time.Second, 60*time.Minute, true, func(ctx context.Context) (bool, error) {
+		logrus.Debug("Checking load balancer host is active \n")
+		stdOut, _, err = oc.AsAdmin().WithoutNamespace().RunInMonitorTest("debug").Args(nodeTgt, "--", "dig", "+short", "+notcp", tcpService.Status.LoadBalancer.Ingress[0].Hostname).Outputs()
+		if err != nil {
+			return false, nil
+		}
+		output := strings.TrimSpace(stdOut)
+		if output == "" {
+			logrus.Debug("Waiting for the load balancer to become active")
+			return false, nil
+		}
+		logrus.Debug("Load balancer active")
+		return true, nil
+	})
+	return err
+}

test/extended/util/client.go

@@ -180,6 +180,32 @@ func NewCLIWithoutNamespace(project string) *CLI {
 	return cli
 }
 
+// NewCLIForMonitorTest initializes the CLI and Kube framework helpers
+// without a namespace. Should be called outside of a Ginkgo .It()
+// function.
+func NewCLIForMonitorTest(project string) *CLI {
+	cli := &CLI{
+		kubeFramework: &framework.Framework{
+			SkipNamespaceCreation: true,
+			BaseName:              project,
+			Options: framework.Options{
+				ClientQPS:   20,
+				ClientBurst: 50,
+			},
+			Timeouts: framework.NewTimeoutContext(),
+		},
+		username:                "admin",
+		execPath:                "oc",
+		adminConfigPath:         KubeConfigPath(),
+		staticConfigManifestDir: StaticConfigManifestDir(),
+		withoutNamespace:        true,
+	}
+
+	// Called only once (assumed the objects will never get modified)
+	cli.setupStaticConfigsFromManifests()
+	return cli
+}
+
 // NewHypershiftManagementCLI returns a CLI that interacts with the Hypershift management cluster.
 // Contrary to a normal CLI it does not perform any cleanup, and it must not be used for any mutating
 // operations. Also, contrary to a normal CLI it must be constructed inside an `It` block. This is
@@ -818,6 +844,22 @@ func (c *CLI) Run(commands ...string) *CLI {
 	return nc.setOutput(c.stdout)
 }
 
+// Executes with the kubeconfig specified from the environment
+func (c *CLI) RunInMonitorTest(commands ...string) *CLI {
+	in, out, errout := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{}
+	nc := &CLI{
+		execPath:        c.execPath,
+		verb:            commands[0],
+		kubeFramework:   c.KubeFramework(),
+		adminConfigPath: c.adminConfigPath,
+		configPath:      c.configPath,
+		username:        c.username,
+		globalArgs:      commands,
+	}
+	nc.stdin, nc.stdout, nc.stderr = in, out, errout
+	return nc.setOutput(c.stdout)
+}
+
 // InputString adds expected input to the command
 func (c *CLI) InputString(input string) *CLI {
 	c.stdin.WriteString(input)