diff --git a/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go b/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go index bdf959cb9212..a79af198eb8f 100644 --- a/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go +++ b/pkg/monitortests/network/disruptionserviceloadbalancer/monitortest.go @@ -9,9 +9,11 @@ import ( "net/http" "os" "strconv" + "strings" "time" "github.com/openshift/origin/pkg/monitortestframework" + "github.com/sirupsen/logrus" configv1 "github.com/openshift/api/config/v1" configclient "github.com/openshift/client-go/config/clientset/versioned" @@ -201,6 +203,15 @@ func (w *availability) StartCollection(ctx context.Context, adminRESTConfig *res return fmt.Errorf("error creating PDB: %w", err) } + // On certain platforms hitting the hostname before it is ready leads to a blackhole, this code checks + // the host from the cluster's context + if infra.Spec.PlatformSpec.Type == configv1.PowerVSPlatformType || infra.Spec.PlatformSpec.Type == configv1.IBMCloudPlatformType { + nodeTgt := "node/" + nodeList.Items[0].ObjectMeta.Name + if err := checkHostnameReady(ctx, tcpService, nodeTgt, w.namespaceName); err != nil { + return err + } + } + // Hit it once before considering ourselves ready fmt.Fprintf(os.Stderr, "hitting pods through the service's LoadBalancer\n") timeout := 10 * time.Minute @@ -315,3 +326,29 @@ func httpGetNoConnectionPoolTimeout(url string, timeout time.Duration) (*http.Re return client.Get(url) } + +// Uses the first node in the cluster to verify the LoadBalancer host is active before returning +func checkHostnameReady(ctx context.Context, tcpService *corev1.Service, nodeTgt string, namespace string) error { + oc := exutil.NewCLIForMonitorTest(tcpService.GetObjectMeta().GetNamespace()) + + var ( + stdOut string + err error + ) + + wait.PollUntilContextTimeout(ctx, 15*time.Second, 60*time.Minute, true, func(ctx context.Context) (bool, error) { + logrus.Debug("Checking load balancer host is active \n") + stdOut, _, err = oc.AsAdmin().WithoutNamespace().RunInMonitorTest("debug").Args(nodeTgt, "--to-namespace"+namespace, "--", "dig", "+short", "+notcp", tcpService.Status.LoadBalancer.Ingress[0].Hostname).Outputs() + if err != nil { + return false, nil + } + output := strings.TrimSpace(stdOut) + if output == "" { + logrus.Debug("Waiting for the load balancer to become active") + return false, nil + } + logrus.Debug("Load balancer active") + return true, nil + }) + return err +} diff --git a/test/extended/util/client.go b/test/extended/util/client.go index 053f2b86f014..2a9d573563a3 100644 --- a/test/extended/util/client.go +++ b/test/extended/util/client.go @@ -179,6 +179,32 @@ func NewCLIWithoutNamespace(project string) *CLI { return cli } +// NewCLIForMonitorTest initializes the CLI and Kube framework helpers +// without a namespace. Should be called outside of a Ginkgo .It() +// function. +func NewCLIForMonitorTest(project string) *CLI { + cli := &CLI{ + kubeFramework: &framework.Framework{ + SkipNamespaceCreation: true, + BaseName: project, + Options: framework.Options{ + ClientQPS: 20, + ClientBurst: 50, + }, + Timeouts: framework.NewTimeoutContext(), + }, + username: "admin", + execPath: "oc", + adminConfigPath: KubeConfigPath(), + staticConfigManifestDir: StaticConfigManifestDir(), + withoutNamespace: true, + } + + // Called only once (assumed the objects will never get modified) + cli.setupStaticConfigsFromManifests() + return cli +} + // NewHypershiftManagementCLI returns a CLI that interacts with the Hypershift management cluster. // Contrary to a normal CLI it does not perform any cleanup, and it must not be used for any mutating // operations. Also, contrary to a normal CLI it must be constructed inside an `It` block. This is @@ -813,6 +839,22 @@ func (c *CLI) Run(commands ...string) *CLI { return nc.setOutput(c.stdout) } +// Executes with the kubeconfig specified from the environment +func (c *CLI) RunInMonitorTest(commands ...string) *CLI { + in, out, errout := &bytes.Buffer{}, &bytes.Buffer{}, &bytes.Buffer{} + nc := &CLI{ + execPath: c.execPath, + verb: commands[0], + kubeFramework: c.KubeFramework(), + adminConfigPath: c.adminConfigPath, + configPath: c.configPath, + username: c.username, + globalArgs: commands, + } + nc.stdin, nc.stdout, nc.stderr = in, out, errout + return nc.setOutput(c.stdout) +} + // InputString adds expected input to the command func (c *CLI) InputString(input string) *CLI { c.stdin.WriteString(input)