Skip to content

Commit

Permalink
improve waiting for resources to exist (#1400)
Browse files Browse the repository at this point in the history 
* improve waiting for resources to exist

* Update test/acceptance/test/utils.go

Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

* Update test/acceptance/test/utils.go

Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>

* fix lint

* fix multi-cluster support and assert gitrepositoies don't exist
correctly

Co-authored-by: Gergely Brautigam <182850+Skarlso@users.noreply.github.com>
  • Loading branch information
@Skarlso
Jake Klein and Skarlso committed Feb 8, 2022
1 parent 83ef33c commit b091349
Show file tree
Hide file tree
Showing 4 changed files with 52 additions and 68 deletions.
12 changes: 6 additions & 6 deletions test/acceptance/test/add_tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -895,8 +895,8 @@ var _ = Describe("Weave GitOps Add App Tests", func() {

By("Then I should see my workload deployed for app2", func() {
verifyWegoAddCommand(appName2, WEGO_DEFAULT_NAMESPACE)
Expect(waitForResource("apps", appName2, WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("configmaps", "helloworld-configmap", WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("apps", appName2, WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("configmaps", "helloworld-configmap", WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)
})

By("When I run gitops add app command for app3: "+appName3, func() {
Expand Down Expand Up @@ -1216,8 +1216,8 @@ var _ = Describe("Weave GitOps Add App Tests", func() {

By("Then I should see my workload deployed to the cluster", func() {
verifyWegoAddCommand(appName, WEGO_DEFAULT_NAMESPACE)
Expect(waitForResource("apps", appName, WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("configmaps", "helloworld-configmap", WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("apps", appName, WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("configmaps", "helloworld-configmap", WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)
})

By("And repo created has private visibility", func() {
Expand Down Expand Up @@ -1281,8 +1281,8 @@ var _ = Describe("Weave GitOps Add App Tests", func() {

By("Then I should see my workload deployed to the cluster", func() {
verifyWegoAddCommand(appName, WEGO_DEFAULT_NAMESPACE)
Expect(waitForResource("apps", appName, WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("configmaps", "helloworld-configmap", WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("apps", appName, WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("configmaps", "helloworld-configmap", WEGO_DEFAULT_NAMESPACE, INSTALL_PODS_READY_TIMEOUT)
})

})
Expand Down
6 changes: 4 additions & 2 deletions test/acceptance/test/scripts/kind-cluster.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@

set -o errexit

echo "Delete existing kind clusters"
kind delete clusters --all
if [[ -z "${SKIP_DELETE}" ]]; then
echo "Delete existing kind clusters"
kind delete clusters --all
fi

echo "Create a new kind cluster with name "$1

Expand Down
4 changes: 0 additions & 4 deletions test/acceptance/test/scripts/kind-multi-cluster.sh
View file

This file was deleted.

98 changes: 42 additions & 56 deletions test/acceptance/test/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,7 +254,7 @@ func ResetOrCreateClusterWithName(namespace string, deleteWegoRuntime bool, clus

var err error
if keepExistingClusters {
err = runCommandPassThrough([]string{}, "./scripts/kind-multi-cluster.sh", kindCluster, "kindest/node:v"+k8sVersion)
err = runCommandPassThrough([]string{"SKIP_DELETE=true"}, "./scripts/kind-cluster.sh", kindCluster, "kindest/node:v"+k8sVersion)
} else {
err = runCommandPassThrough([]string{}, "./scripts/kind-cluster.sh", kindCluster, "kindest/node:v"+k8sVersion)
}
Expand Down Expand Up @@ -339,39 +339,30 @@ func getGitRepoVisibility(org string, repo string, providerName gitproviders.Git
return visibility
}

func waitForResource(resourceType string, resourceName string, namespace string, timeout time.Duration) error {
pollInterval := 5

if timeout < 5*time.Second {
timeout = 5 * time.Second
}

timeoutInSeconds := int(timeout.Seconds())
for i := pollInterval; i < timeoutInSeconds; i += pollInterval {
log.Infof("Waiting for %s in namespace: %s... : %d second(s) passed of %d seconds timeout", resourceType+"/"+resourceName, namespace, i, timeoutInSeconds)
err := runCommandPassThroughWithoutOutput([]string{}, "sh", "-c", fmt.Sprintf("kubectl get %s %s -n %s", resourceType, resourceName, namespace))

if err == nil {
log.Infof("%s is available in cluster", resourceType+"/"+resourceName)
command := exec.Command("sh", "-c", fmt.Sprintf("kubectl get %s %s -n %s", resourceType, resourceName, namespace))
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).ShouldNot(HaveOccurred())
Eventually(session).Should(gexec.Exit())

noResourcesFoundMessage := fmt.Sprintf("No resources found in %s namespace", namespace)

if strings.Contains(string(session.Wait().Out.Contents()), noResourcesFoundMessage) {
log.Infof("Got message => {" + noResourcesFoundMessage + "} Continue looking for resource(s)")
continue
}

return nil
//Assumes resource will eventually contain a status.Conditions where Type=Ready exists
func waitForResourceToBeReady(resourceType string, resourceName string, namespace string, timeout time.Duration) {
EventuallyWithOffset(1, func() error {
log.Infof("Waiting for %s/%s in namespace: %q to be ready : ", resourceType, resourceName, namespace)
kubectlCommand := fmt.Sprintf("kubectl -n %s wait --for=condition=ready %s %s", namespace, resourceType, resourceName)
if resourceName == "" {
kubectlCommand = kubectlCommand + " --all"
}
if _, err := exec.Command("sh", "-c", kubectlCommand).CombinedOutput(); err != nil {
return err
}
return nil
}, timeout, "5s").Should(Succeed(), fmt.Sprintf("Failed to find the resource %s of type %s, timeout reached", resourceName, resourceType))
}

time.Sleep(time.Duration(pollInterval) * time.Second)
}

return fmt.Errorf("Error: Failed to find the resource %s of type %s, timeout reached", resourceName, resourceType)
func waitForResourceToExist(resourceType string, resourceName string, namespace string, timeout time.Duration) {
EventuallyWithOffset(1, func() error {
log.Infof("Waiting for %s/%s in namespace: %q to exist: ", resourceType, resourceName, namespace)
command := exec.Command("sh", "-c", fmt.Sprintf("kubectl get %s %s -n %s", resourceType, resourceName, namespace))
if _, err := command.CombinedOutput(); err != nil {
return err
}
return nil
}, timeout, "5s").Should(Succeed(), fmt.Sprintf("Failed to find the resource %s of type %s, timeout reached", resourceName, resourceType))
}

func waitForNamespaceToTerminate(namespace string, timeout time.Duration) error {
Expand Down Expand Up @@ -414,14 +405,14 @@ func waitForNamespaceToTerminate(namespace string, timeout time.Duration) error
}

func VerifyControllersInCluster(namespace string) {
Expect(waitForResource("deploy", "helm-controller", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("deploy", "kustomize-controller", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("deploy", "notification-controller", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("deploy", "source-controller", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("deploy", "image-automation-controller", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("deploy", "image-reflector-controller", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("deploy", "wego-app", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("pods", "", namespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("deploy", "helm-controller", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("deploy", "kustomize-controller", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("deploy", "notification-controller", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("deploy", "source-controller", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("deploy", "image-automation-controller", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("deploy", "image-reflector-controller", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToExist("deploy", "wego-app", namespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToBeReady("pods", "", namespace, INSTALL_PODS_READY_TIMEOUT)

By("And I wait for the gitops controllers to be ready", func() {
command := exec.Command("sh", "-c", fmt.Sprintf("kubectl wait --for=condition=Ready --timeout=%s -n %s --all pod --selector='app!=wego-app'", "120s", namespace))
Expand Down Expand Up @@ -579,7 +570,7 @@ func waitForAppRemoval(appName string, timeout time.Duration) error {
func runCommandPassThrough(env []string, name string, arg ...string) error {
cmd := exec.Command(name, arg...)
if len(env) > 0 {
cmd.Env = env
cmd.Env = append(os.Environ(), env...)
}

cmd.Stdout = os.Stdout
Expand All @@ -588,15 +579,6 @@ func runCommandPassThrough(env []string, name string, arg ...string) error {
return cmd.Run()
}

func runCommandPassThroughWithoutOutput(env []string, name string, arg ...string) error {
cmd := exec.Command(name, arg...)
if len(env) > 0 {
cmd.Env = env
}

return cmd.Run()
}

func runCommandAndReturnStringOutput(commandToRun string) (stdOut string, stdErr string) {
command := exec.Command("sh", "-c", commandToRun)
session, _ := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expand Down Expand Up @@ -635,36 +617,40 @@ func verifyWegoAddCommand(appName string, wegoNamespace string) {
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).ShouldNot(HaveOccurred())
Eventually(session, INSTALL_PODS_READY_TIMEOUT).Should(gexec.Exit())
Expect(waitForResource("GitRepositories", appName, wegoNamespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("GitRepositories", appName, wegoNamespace, INSTALL_PODS_READY_TIMEOUT)
}

func verifyWegoHelmAddCommand(appName string, wegoNamespace string) {
command := exec.Command("sh", "-c", fmt.Sprintf("kubectl wait --for=condition=Ready --timeout=60s -n %s HelmRepositories --all", wegoNamespace))
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).ShouldNot(HaveOccurred())
Eventually(session, INSTALL_PODS_READY_TIMEOUT).Should(gexec.Exit())
Expect(waitForResource("HelmRepositories", appName, wegoNamespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("HelmRepositories", appName, wegoNamespace, INSTALL_PODS_READY_TIMEOUT)
}

func verifyWegoAddCommandWithDryRun(appRepoName string, wegoNamespace string) {
command := exec.Command("sh", "-c", fmt.Sprintf("kubectl wait --for=condition=Ready --timeout=30s -n %s GitRepositories --all", wegoNamespace))
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).ShouldNot(HaveOccurred())
Eventually(session, INSTALL_PODS_READY_TIMEOUT).Should(gexec.Exit())
Expect(waitForResource("GitRepositories", appRepoName, wegoNamespace, THIRTY_SECOND_TIMEOUT)).ToNot(Succeed())

command = exec.Command("sh", "-c", fmt.Sprintf("kubectl get GitRepositories %s -n %s", appRepoName, wegoNamespace))
out, err := command.CombinedOutput()
Expect(err).To(HaveOccurred())
Expect(string(out)).To(ContainSubstring("not found"))
}

func verifyWorkloadIsDeployed(workloadName string, workloadNamespace string) {
Expect(waitForResource("deploy", workloadName, workloadNamespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
Expect(waitForResource("pods", "", workloadNamespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToExist("deploy", workloadName, workloadNamespace, INSTALL_PODS_READY_TIMEOUT)
waitForResourceToBeReady("pods", "", workloadNamespace, INSTALL_PODS_READY_TIMEOUT)
command := exec.Command("sh", "-c", fmt.Sprintf("kubectl wait --for=condition=Ready --timeout=60s -n %s --all pods --selector='app!=wego-app'", workloadNamespace))
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expect(err).ShouldNot(HaveOccurred())
Eventually(session, INSTALL_PODS_READY_TIMEOUT).Should(gexec.Exit())
}

func verifyHelmPodWorkloadIsDeployed(workloadName string, workloadNamespace string) {
Expect(waitForResource("pods", workloadName, workloadNamespace, INSTALL_PODS_READY_TIMEOUT)).To(Succeed())
waitForResourceToBeReady("pods", workloadName, workloadNamespace, INSTALL_PODS_READY_TIMEOUT)
c := fmt.Sprintf("kubectl wait --for=condition=Ready --timeout=360s -n %s --all pods --selector='app!=wego-app'", workloadNamespace)
command := exec.Command("sh", "-c", c)
session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
Expand Down

0 comments on commit b091349

Please sign in to comment.