Skip to content

Commit

Permalink
[Cherry-pick for 1.7.0] (#87)
Browse files Browse the repository at this point in the history 
* chore(probe): Adding probes in all go experiments (#80)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

* (fix)resource-chaoslib: run command with a shell instance (#81)

Signed-off-by: ksatchit <karthik.s@mayadata.io>

* refactor(go-experiments): Refactor all the go experiments (#82)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

* chore(exp): Add pod autoscaler experiment used to check the scalability of the application pod (#65)

* chore(exp): Add pod autoscaler experiment used to check the salability of the application pod

Signed-off-by: Udit Gaurav <uditgaurav@gmail.com>

* Adding abort in the experiment

Signed-off-by: Udit Gaurav <udit.gaurav@mayadata.io>

* update(chaosresult): updating the chaosresult for probe score (#85)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

* refactor(pod-autoscaler): refactor the pod-scaler experiment (#86)

Signed-off-by: shubhamchaudhary <shubham.chaudhary@mayadata.io>

Co-authored-by: Shubham Chaudhary <shubham.chaudhary@mayadata.io>
Co-authored-by: UDIT GAURAV <35391335+uditgaurav@users.noreply.github.com>
  • Loading branch information
@ispeakc0de @uditgaurav
3 people committed Aug 15, 2020
1 parent 5117d45 commit b7d8596
Show file tree
Hide file tree
Showing 58 changed files with 1,886 additions and 643 deletions.
2 changes: 2 additions & 0 deletions build/generate_go_binary
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,5 +26,7 @@ go build -o build/_output/node-cpu-hog ./experiments/generic/node-cpu-hog
go build -o build/_output/container-kill ./experiments/generic/container-kill
# Buiding go binaries for disk_fill experiment
go build -o build/_output/disk-fill ./experiments/generic/disk-fill
# Buiding go binaries for pod-autoscaler experiment
go build -o build/_output/pod-autoscaler ./experiments/generic/pod-autoscaler
# Buiding go binaries for container_kill helper
go build -o build/_output/container-killer ./chaoslib/litmus/container_kill/helper
63 changes: 12 additions & 51 deletions chaoslib/litmus/container_kill/container-kill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/litmuschaos/litmus-go/pkg/math"
"github.com/litmuschaos/litmus-go/pkg/status"
"github.com/litmuschaos/litmus-go/pkg/types"
"github.com/openebs/maya/pkg/util/retry"
"github.com/litmuschaos/litmus-go/pkg/utils/common"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
apiv1 "k8s.io/api/core/v1"
Expand All @@ -24,7 +24,7 @@ func PrepareContainerKill(experimentsDetails *experimentTypes.ExperimentDetails,
//Select application pod and node name for the container-kill
appName, appNodeName, err := GetApplicationPod(experimentsDetails, clients)
if err != nil {
return errors.Errorf("Unable to get the application name and application nodename due to, err: %v", err)
return errors.Errorf("Unable to get the application pod and node name due to, err: %v", err)
}

//Get the target container name of the application pod
Expand All @@ -45,7 +45,7 @@ func PrepareContainerKill(experimentsDetails *experimentTypes.ExperimentDetails,
GetIterations(experimentsDetails)

// generating a unique string which can be appended with the helper pod name & labels for the uniquely identification
experimentsDetails.RunID = GetRunID()
experimentsDetails.RunID = common.GetRunID()

// Getting the serviceAccountName, need permission inside helper pod to create the events
if experimentsDetails.ChaosServiceAccount == "" {
Expand All @@ -58,7 +58,7 @@ func PrepareContainerKill(experimentsDetails *experimentTypes.ExperimentDetails,
//Waiting for the ramp time before chaos injection
if experimentsDetails.RampTime != 0 {
log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", strconv.Itoa(experimentsDetails.RampTime))
waitForDuration(experimentsDetails.RampTime)
common.WaitForDuration(experimentsDetails.RampTime)
}

// creating the helper pod to perform container kill chaos
Expand All @@ -69,30 +69,30 @@ func PrepareContainerKill(experimentsDetails *experimentTypes.ExperimentDetails,

//checking the status of the helper pod, wait till the helper pod comes to running state else fail the experiment
log.Info("[Status]: Checking the status of the helper pod")
err = status.CheckApplicationStatus(experimentsDetails.ChaosNamespace, "name=container-kill-"+experimentsDetails.RunID, experimentsDetails.Timeout, experimentsDetails.Delay, clients)
err = status.CheckApplicationStatus(experimentsDetails.ChaosNamespace, "name="+experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, experimentsDetails.Timeout, experimentsDetails.Delay, clients)
if err != nil {
return errors.Errorf("helper pod is not in running state, err: %v", err)
}

// Wait till the completion of the helper pod
// set an upper limit for the waiting time
log.Info("[Wait]: waiting till the completion of the helper pod")
podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name=container-kill-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+experimentsDetails.ChaosInterval+60, "container-kill")
podStatus, err := status.WaitForCompletion(experimentsDetails.ChaosNamespace, "name="+experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, clients, experimentsDetails.ChaosDuration+experimentsDetails.ChaosInterval+60, experimentsDetails.ExperimentName)
if err != nil || podStatus == "Failed" {
return errors.Errorf("helper pod failed due to, err: %v", err)
}

//Deleting the helper pod for container-kill chaos
log.Info("[Cleanup]: Deleting the helper pod")
err = DeleteHelperPod(experimentsDetails, clients)
err = common.DeletePod(experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, "name="+experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients)
if err != nil {
return errors.Errorf("Unable to delete the helper pod, err: %v", err)
}

//Waiting for the ramp time after chaos injection
if experimentsDetails.RampTime != 0 {
log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", strconv.Itoa(experimentsDetails.RampTime))
waitForDuration(experimentsDetails.RampTime)
common.WaitForDuration(experimentsDetails.RampTime)
}
return nil
}
Expand All @@ -107,21 +107,6 @@ func GetIterations(experimentsDetails *experimentTypes.ExperimentDetails) {

}

//waitForDuration waits for the given time duration (in seconds)
func waitForDuration(duration int) {
time.Sleep(time.Duration(duration) * time.Second)
}

// GetRunID generate a random string
func GetRunID() string {
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz")
runID := make([]rune, 6)
for i := range runID {
runID[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(runID)
}

// GetServiceAccount find the serviceAccountName for the helper pod
func GetServiceAccount(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets) error {
pod, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Get(experimentsDetails.ChaosPodName, v1.GetOptions{})
Expand Down Expand Up @@ -166,11 +151,11 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie

helperPod := &apiv1.Pod{
ObjectMeta: v1.ObjectMeta{
Name: "container-kill-" + experimentsDetails.RunID,
Name: experimentsDetails.ExperimentName + "-" + experimentsDetails.RunID,
Namespace: experimentsDetails.ChaosNamespace,
Labels: map[string]string{
"app": "container-kill",
"name": "container-kill-" + experimentsDetails.RunID,
"app": experimentsDetails.ExperimentName,
"name": experimentsDetails.ExperimentName + "-" + experimentsDetails.RunID,
"chaosUID": string(experimentsDetails.ChaosUID),
},
},
Expand Down Expand Up @@ -198,7 +183,7 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
},
Containers: []apiv1.Container{
{
Name: "container-kill",
Name: experimentsDetails.ExperimentName,
Image: experimentsDetails.LIBImage,
ImagePullPolicy: apiv1.PullAlways,
Command: []string{
Expand Down Expand Up @@ -232,30 +217,6 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie

}

//DeleteHelperPod deletes the helper pod and wait until it got terminated
func DeleteHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets) error {

err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Delete("container-kill-"+experimentsDetails.RunID, &v1.DeleteOptions{})

if err != nil {
return err
}

// waiting for the termination of the pod
err = retry.
Times(uint(experimentsDetails.Timeout / experimentsDetails.Delay)).
Wait(time.Duration(experimentsDetails.Delay) * time.Second).
Try(func(attempt uint) error {
podSpec, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).List(v1.ListOptions{LabelSelector: "name=container-kill-" + experimentsDetails.RunID})
if err != nil || len(podSpec.Items) != 0 {
return errors.Errorf("Pod is not deleted yet, err: %v", err)
}
return nil
})

return err
}

// GetPodEnv derive all the env required for the helper pod
func GetPodEnv(experimentsDetails *experimentTypes.ExperimentDetails, podName string) []apiv1.EnvVar {

Expand Down
68 changes: 15 additions & 53 deletions chaoslib/litmus/disk_fill/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import (
"github.com/litmuschaos/litmus-go/pkg/log"
"github.com/litmuschaos/litmus-go/pkg/status"
"github.com/litmuschaos/litmus-go/pkg/types"
"github.com/litmuschaos/litmus-go/pkg/utils/common"
"github.com/litmuschaos/litmus-go/pkg/utils/exec"
"github.com/openebs/maya/pkg/util/retry"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
apiv1 "k8s.io/api/core/v1"
Expand All @@ -31,7 +31,7 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie
//Select application pod & node for the disk fill chaos
appName, appNodeName, err := GetApplicationPod(experimentsDetails, clients)
if err != nil {
return errors.Errorf("Unable to get the application name and application nodename due to, err: %v", err)
return errors.Errorf("Unable to get the application pod and node name due to, err: %v", err)
}

//Get the target container name of the application pod
Expand Down Expand Up @@ -64,12 +64,12 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie
})

// generating a unique string which can be appended with the helper pod name & labels for the uniquely identification
experimentsDetails.RunID = GetRunID()
experimentsDetails.RunID = common.GetRunID()

//Waiting for the ramp time before chaos injection
if experimentsDetails.RampTime != 0 {
log.Infof("[Ramp]: Waiting for the %vs ramp time before injecting chaos", strconv.Itoa(experimentsDetails.RampTime))
waitForDuration(experimentsDetails.RampTime)
common.WaitForDuration(experimentsDetails.RampTime)
}

// generating the chaos inject event in the chaosengine
Expand All @@ -87,7 +87,7 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie

//checking the status of the helper pod, wait till the helper pod comes to running state else fail the experiment
log.Info("[Status]: Checking the status of the helper pod")
err = status.CheckApplicationStatus(experimentsDetails.ChaosNamespace, "name=disk-fill-"+experimentsDetails.RunID, experimentsDetails.Timeout, experimentsDetails.Delay, clients)
err = status.CheckApplicationStatus(experimentsDetails.ChaosNamespace, "name="+experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, experimentsDetails.Timeout, experimentsDetails.Delay, clients)
if err != nil {
return errors.Errorf("helper pod is not in running state, err: %v", err)
}
Expand All @@ -100,6 +100,7 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie
if err != nil {
return errors.Errorf("Unable to get ephemeral storage details due to err: %v", err)
}

// filtering out the used ephemeral storage from the output of du command
usedEphemeralStorageSize, err := FilterUsedEphemeralStorage(ephemeralStorageDetails)
if err != nil {
Expand All @@ -125,7 +126,7 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie

// waiting for the chaos duration
log.Infof("[Wait]: Waiting for the %vs after injecting chaos", strconv.Itoa(experimentsDetails.ChaosDuration))
waitForDuration(experimentsDetails.ChaosDuration)
common.WaitForDuration(experimentsDetails.ChaosDuration)

// It will delete the target pod if target pod is evicted
// if target pod is still running then it will delete all the files, which was created earlier during chaos execution
Expand All @@ -136,15 +137,15 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie

//Deleting the helper pod
log.Info("[Cleanup]: Deleting the helper pod")
err = DeleteHelperPod(experimentsDetails, clients, experimentsDetails.RunID)
err = common.DeletePod(experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, "name="+experimentsDetails.ExperimentName+"-"+experimentsDetails.RunID, experimentsDetails.ChaosNamespace, chaosDetails.Timeout, chaosDetails.Delay, clients)
if err != nil {
errors.Errorf("Unable to delete the helper pod, err: %v", err)
return errors.Errorf("Unable to delete the helper pod, err: %v", err)
}

//Waiting for the ramp time after chaos injection
if experimentsDetails.RampTime != 0 {
log.Infof("[Ramp]: Waiting for the %vs ramp time after injecting chaos", strconv.Itoa(experimentsDetails.RampTime))
waitForDuration(experimentsDetails.RampTime)
common.WaitForDuration(experimentsDetails.RampTime)
}
return nil
}
Expand Down Expand Up @@ -176,21 +177,6 @@ func GetTargetContainer(experimentsDetails *experimentTypes.ExperimentDetails, a
return pod.Spec.Containers[0].Name, nil
}

//waitForDuration waits for the given time duration (in seconds)
func waitForDuration(duration int) {
time.Sleep(time.Duration(duration) * time.Second)
}

// GetRunID generate a random string
func GetRunID() string {
var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz")
runID := make([]rune, 6)
for i := range runID {
runID[i] = letterRunes[rand.Intn(len(letterRunes))]
}
return string(runID)
}

// CreateHelperPod derive the attributes for helper pod and create the helper pod
func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, appName, appNodeName string) error {

Expand All @@ -199,11 +185,11 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie

helperPod := &apiv1.Pod{
ObjectMeta: v1.ObjectMeta{
Name: "disk-fill-" + experimentsDetails.RunID,
Name: experimentsDetails.ExperimentName + "-" + experimentsDetails.RunID,
Namespace: experimentsDetails.ChaosNamespace,
Labels: map[string]string{
"app": "disk-fill",
"name": "disk-fill-" + experimentsDetails.RunID,
"app": experimentsDetails.ExperimentName,
"name": experimentsDetails.ExperimentName + "-" + experimentsDetails.RunID,
"chaosUID": string(experimentsDetails.ChaosUID),
},
},
Expand All @@ -222,7 +208,7 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
},
Containers: []apiv1.Container{
{
Name: "disk-fill",
Name: experimentsDetails.ExperimentName,
Image: experimentsDetails.LIBImage,
ImagePullPolicy: apiv1.PullAlways,
Args: []string{
Expand All @@ -248,30 +234,6 @@ func CreateHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
return err
}

//DeleteHelperPod deletes the helper pod and wait until it got terminated
func DeleteHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, runID string) error {

err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Delete("disk-fill-"+runID, &v1.DeleteOptions{})

if err != nil {
return err
}

// waiting for the termination of the pod
err = retry.
Times(uint(experimentsDetails.Timeout / experimentsDetails.Delay)).
Wait(time.Duration(experimentsDetails.Delay) * time.Second).
Try(func(attempt uint) error {
podSpec, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).List(v1.ListOptions{LabelSelector: "name=disk-fill-" + runID})
if err != nil || len(podSpec.Items) != 0 {
return errors.Errorf("Helper Pod is not deleted yet, err: %v", err)
}
return nil
})

return err
}

// GetEphemeralStorageAttributes derive the ephemeral storage attributes from the target pod
func GetEphemeralStorageAttributes(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, podName string) (int64, int64, error) {

Expand Down Expand Up @@ -367,7 +329,7 @@ func Remedy(experimentsDetails *experimentTypes.ExperimentDetails, clients clien
command := "rm -rf /diskfill/" + containerID + "/diskfill"
_, err = exec.Exec(execCommandDetails, clients, strings.Fields(command))
if err != nil {
errors.Errorf("Unable to delete files to clean ephemeral storage due to err: %v", err)
return errors.Errorf("Unable to delete files to clean ephemeral storage due to err: %v", err)
}
}
return nil
Expand Down
Loading

0 comments on commit b7d8596

Please sign in to comment.