Skip to content
Permalink
Browse files

Wait for rollout (#2731)

* wait for deployment rollout

* add integration test to cover waiting for deployment

cleanup kubectl

* wrap error

* fix typos

* fix tests and address review comments
  • Loading branch information
kadel committed Mar 25, 2020
1 parent 796991d commit 38a55493fbe0b5f58eb0219738ea283ff8d6d4d7
@@ -51,6 +51,11 @@ func (a Adapter) Push(path string, ignoredFiles []string, forceBuild bool, globE
return errors.Wrap(err, "unable to create or update component")
}

_, err = a.Client.WaitForDeploymentRollout(a.ComponentName)
if err != nil {
return errors.Wrap(err, "error while waiting for deployment rollout")
}

// Sync source code to the component
// If syncing for the first time, sync the entire source directory
// If syncing to an already running component, sync the deltas
@@ -1,6 +1,10 @@
package kclient

import (
"fmt"
"time"

"github.com/golang/glog"
"github.com/pkg/errors"
appsv1 "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -10,6 +14,10 @@ import (
const (
DeploymentKind = "Deployment"
DeploymentAPIVersion = "apps/v1"

// TimedOutReason is added in a deployment when its newest replica set fails to show any progress
// within the given deadline (progressDeadlineSeconds).
timedOutReason = "ProgressDeadlineExceeded"
)

// GetDeploymentByName gets a deployment by querying by name
@@ -18,6 +26,76 @@ func (c *Client) GetDeploymentByName(name string) (*appsv1.Deployment, error) {
return deployment, err
}

// getDeploymentCondition returns the condition with the provided type
// from https://github.com/kubernetes/kubectl/blob/8bc20f428d7d5aed031de5fa160081de7b5af2b0/pkg/util/deployment/deployment.go#L58
func getDeploymentCondition(status appsv1.DeploymentStatus, condType appsv1.DeploymentConditionType) *appsv1.DeploymentCondition {
for i := range status.Conditions {
c := status.Conditions[i]
if c.Type == condType {
return &c
}
}
return nil
}

// WaitForDeploymentRollout waits for deployment to finish rollout. Returns the state of the deployment after rollout.
func (c *Client) WaitForDeploymentRollout(deploymentName string) (*appsv1.Deployment, error) {
glog.V(4).Infof("Waiting for %s deployment roll out", deploymentName)

w, err := c.KubeClient.AppsV1().Deployments(c.Namespace).Watch(metav1.ListOptions{FieldSelector: "metadata.name=" + deploymentName})
if err != nil {
return nil, errors.Wrapf(err, "unable to watch deployment")
}
defer w.Stop()

success := make(chan *appsv1.Deployment)
failure := make(chan error)

go func() {
defer close(success)
defer close(failure)

for {
val, ok := <-w.ResultChan()
if !ok {
failure <- errors.New("watch channel was closed")
return
}
//based on https://github.com/kubernetes/kubectl/blob/9a3954bf653c874c8af6f855f2c754a8e1a44b9e/pkg/polymorphichelpers/rollout_status.go#L66-L91
if deployment, ok := val.Object.(*appsv1.Deployment); ok {
if deployment.Generation <= deployment.Status.ObservedGeneration {
cond := getDeploymentCondition(deployment.Status, appsv1.DeploymentProgressing)
if cond != nil && cond.Reason == timedOutReason {
failure <- fmt.Errorf("deployment %q exceeded its progress deadline", deployment.Name)
} else if deployment.Spec.Replicas != nil && deployment.Status.UpdatedReplicas < *deployment.Spec.Replicas {
glog.V(4).Infof("Waiting for deployment %q rollout to finish: %d out of %d new replicas have been updated...\n", deployment.Name, deployment.Status.UpdatedReplicas, *deployment.Spec.Replicas)
} else if deployment.Status.Replicas > deployment.Status.UpdatedReplicas {
glog.V(4).Infof("Waiting for deployment %q rollout to finish: %d old replicas are pending termination...\n", deployment.Name, deployment.Status.Replicas-deployment.Status.UpdatedReplicas)
} else if deployment.Status.AvailableReplicas < deployment.Status.UpdatedReplicas {
glog.V(4).Infof("Waiting for deployment %q rollout to finish: %d of %d updated replicas are available...\n", deployment.Name, deployment.Status.AvailableReplicas, deployment.Status.UpdatedReplicas)
} else {
glog.V(4).Infof("Deployment %q successfully rolled out\n", deployment.Name)
success <- deployment
}
}
glog.V(4).Infof("Waiting for deployment spec update to be observed...\n")

} else {
failure <- errors.New("unable to convert event object to Pod")
}
}
}()

select {
case val := <-success:
return val, nil
case err := <-failure:
return nil, err
case <-time.After(5 * time.Minute):
return nil, errors.Errorf("timeout while waiting for %s deployment roll out", deploymentName)
}
}

// CreateDeployment creates a deployment based on the given deployment spec
func (c *Client) CreateDeployment(deploymentSpec appsv1.DeploymentSpec) (*appsv1.Deployment, error) {
// inherit ObjectMeta from deployment spec so that namespace, labels, owner references etc will be the same
@@ -3,26 +3,24 @@ metadata:
name: test-devfile
components:
- type: dockerimage
image: node:stretch
command: ["tail", "-f", "/dev/null"]
image: quay.io/eclipse/che-nodejs10-ubi:nightly
endpoints:
- name: '9090/tcp'
- name: "9090/tcp"
port: 9090
alias: runtime
env:
- name: FOO
value: "bar"
memoryLimit: 1024Mi
mountSources: true
commands:
-
name: build
- name: build
actions:
-
type: exec
- type: exec
component: runtime
command: "npm install"
-
name: run
- name: run
actions:
-
type: exec
- type: exec
component: runtime
command: "node run"
command: "node run"
@@ -37,6 +37,13 @@ func DeleteDir(dir string) {

}

// DeleteFile deletes file
func DeleteFile(filepath string) {
fmt.Fprintf(GinkgoWriter, "Deleting file: %s\n", filepath)
err := os.Remove(filepath)
Expect(err).NotTo(HaveOccurred())
}

// RenameFile renames a file from oldFileName to newFileName
func RenameFile(oldFileName, newFileName string) {
err := os.Rename(oldFileName, newFileName)
@@ -13,18 +13,20 @@ import (
)

var _ = Describe("odo devfile push command tests", func() {
var project string
var namespace string
var context string
var currentWorkingDirectory string

var sourcePath = "/projects"

// TODO: all oc commands in all devfile related test should get replaced by kubectl
// TODO: to goal is not to use "oc"
oc = helper.NewOcRunner("oc")

// This is run after every Spec (It)
var _ = BeforeEach(func() {
SetDefaultEventuallyTimeout(10 * time.Minute)
project = helper.CreateRandProject()
namespace = helper.CreateRandProject()
context = helper.CreateNewDevfileContext()
currentWorkingDirectory = helper.Getwd()
helper.Chdir(context)
@@ -34,7 +36,7 @@ var _ = Describe("odo devfile push command tests", func() {
// Clean up after the test
// This is run after every Spec (It)
var _ = AfterEach(func() {
helper.DeleteProject(project)
helper.DeleteProject(namespace)
helper.Chdir(currentWorkingDirectory)
helper.DeleteDir(context)
os.Unsetenv("GLOBALODOCONFIG")
@@ -48,8 +50,12 @@ var _ = Describe("odo devfile push command tests", func() {

helper.CopyExample(filepath.Join("source", "devfiles", "nodejs"), context)

output := helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
output := helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)
Expect(output).To(ContainSubstring("Changes successfully pushed to component"))

// update devfile and push again
helper.ReplaceString("devfile.yaml", "name: FOO", "name: BAR")
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)
})

})
@@ -62,13 +68,13 @@ var _ = Describe("odo devfile push command tests", func() {

helper.CopyExample(filepath.Join("source", "devfiles", "nodejs"), context)

helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
output := helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)
output := helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)

Expect(output).To(ContainSubstring("No file changes detected, skipping build"))
helper.ReplaceString(filepath.Join(context, "server.js"), "node listening on", "UPDATED!")

helper.CmdShouldPass("odo", "push", "--context", filepath.Join(context, "nodejs-ex"), "--namespace", project)
helper.CmdShouldPass("odo", "push", "--context", filepath.Join(context, "nodejs-ex"), "--namespace", namespace)
})

It("should be able to create a file, push, delete, then push again propagating the deletions", func() {
@@ -88,25 +94,25 @@ var _ = Describe("odo devfile push command tests", func() {
helper.MakeDir(newDirPath)

// Push
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)

// component name is currently equal to directory name until odo create for devfiles is implemented
cmpName := filepath.Base(context)

// Check to see if it's been pushed (foobar.txt abd directory testdir)
podName := oc.GetRunningPodNameByComponent(cmpName, project)
podName := oc.GetRunningPodNameByComponent(cmpName, namespace)

stdOut := oc.ExecListDir(podName, project, sourcePath)
stdOut := oc.ExecListDir(podName, namespace, sourcePath)
Expect(stdOut).To(ContainSubstring(("foobar.txt")))
Expect(stdOut).To(ContainSubstring(("testdir")))

// Now we delete the file and dir and push
helper.DeleteDir(newFilePath)
helper.DeleteDir(newDirPath)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project, "-v4")
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace, "-v4")

// Then check to see if it's truly been deleted
stdOut = oc.ExecListDir(podName, project, sourcePath)
stdOut = oc.ExecListDir(podName, namespace, sourcePath)
Expect(stdOut).To(Not(ContainSubstring(("foobar.txt"))))
Expect(stdOut).To(Not(ContainSubstring(("testdir"))))
})
@@ -116,18 +122,18 @@ var _ = Describe("odo devfile push command tests", func() {
helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true")

helper.CopyExample(filepath.Join("source", "devfiles", "nodejs-multicontainer"), context)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)

// component name is currently equal to directory name until odo create for devfiles is implemented
cmpName := filepath.Base(context)

// Check to see if it's been pushed (foobar.txt abd directory testdir)
podName := oc.GetRunningPodNameByComponent(cmpName, project)
podName := oc.GetRunningPodNameByComponent(cmpName, namespace)

var statErr error
oc.CheckCmdOpInRemoteDevfilePod(
podName,
project,
namespace,
[]string{"stat", "/projects/server.js"},
func(cmdOp string, err error) bool {
statErr = err
@@ -136,11 +142,11 @@ var _ = Describe("odo devfile push command tests", func() {
)
Expect(statErr).ToNot(HaveOccurred())
Expect(os.Remove(filepath.Join(context, "server.js"))).NotTo(HaveOccurred())
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)

oc.CheckCmdOpInRemoteDevfilePod(
podName,
project,
namespace,
[]string{"stat", "/projects/server.js"},
func(cmdOp string, err error) bool {
statErr = err
@@ -156,10 +162,10 @@ var _ = Describe("odo devfile push command tests", func() {
helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true")

helper.CopyExample(filepath.Join("source", "devfiles", "nodejs-multicontainer"), context)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project)
helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace)

// use the force build flag and push
output := helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", project, "-f")
output := helper.CmdShouldPass("odo", "push", "--devfile", "devfile.yaml", "--namespace", namespace, "-f")
Expect(output).To(Not(ContainSubstring("No file changes detected, skipping build")))
})
})

0 comments on commit 38a5549

Please sign in to comment.
You can’t perform that action at this time.