Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Chore(Enhancement): Changed Disk Fill experiment to not require CONTAINER_PATH #605

Merged
merged 2 commits into from
Nov 24, 2022
Merged

Chore(Enhancement): Changed Disk Fill experiment to not require CONTAINER_PATH #605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4802d58
changed dd to use nsenter
avaakash Nov 16, 2022
f43a63a
Merge branch 'master' into disk-fill-helper
ispeakc0de Nov 22, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
51 changes: 31 additions & 20 deletions chaoslib/litmus/disk-fill/helper/disk-fill.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ import (

var inject, abort chan os.Signal

var err error

// Helper injects the disk-fill chaos
func Helper(clients clients.ClientSets) {

var err error

experimentsDetails := experimentTypes.ExperimentDetails{}
eventsDetails := types.EventDetails{}
chaosDetails := types.ChaosDetails{}
Expand Down Expand Up @@ -60,12 +60,12 @@ func Helper(clients clients.ClientSets) {
// Set the chaos result uid
result.SetResultUID(&resultDetails, clients, &chaosDetails)

if err := diskFill(&experimentsDetails, clients, &eventsDetails, &chaosDetails, &resultDetails); err != nil {
if err = diskFill(&experimentsDetails, clients, &eventsDetails, &chaosDetails, &resultDetails); err != nil {
log.Fatalf("helper pod failed, err: %v", err)
}
}

//diskFill contains steps to inject disk-fill chaos
// diskFill contains steps to inject disk-fill chaos
func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails) error {

targetList, err := common.ParseTargets()
Expand All @@ -88,6 +88,12 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
return err
}

// extract out the pid of the target container
td.TargetPID, err = common.GetPID(experimentsDetails.ContainerRuntime, td.ContainerId, experimentsDetails.SocketPath)
if err != nil {
return err
}

td.SizeToFill, err = getDiskSizeToFill(td, experimentsDetails, clients)
if err != nil {
return err
Expand Down Expand Up @@ -119,16 +125,15 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
os.Exit(1)
default:
}

for _, t := range targets {
if t.SizeToFill > 0 {
if err := fillDisk(t.ContainerId, t.SizeToFill, experimentsDetails.DataBlockSize); err != nil {
if err = fillDisk(t.TargetPID, t.SizeToFill, experimentsDetails.DataBlockSize); err != nil {
return err
}
log.Infof("successfully injected chaos on target: {name: %s, namespace: %v, container: %v}", t.Name, t.Namespace, t.TargetContainer)
if err = result.AnnotateChaosResult(resultDetails.Name, chaosDetails.ChaosNamespace, "injected", "pod", t.Name); err != nil {
if remedyErr := remedy(t, clients); remedyErr != nil {
return remedyErr
if revertErr := revertDiskFill(t, clients); revertErr != nil {
return revertErr
}
return err
}
Expand All @@ -148,7 +153,7 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
for _, t := range targets {
// It will delete the target pod if target pod is evicted
// if target pod is still running then it will delete all the files, which was created earlier during chaos execution
err = remedy(t, clients)
err = revertDiskFill(t, clients)
if err != nil {
errList = append(errList, err.Error())
continue
Expand All @@ -166,11 +171,11 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
}

// fillDisk fill the ephemeral disk by creating files
func fillDisk(containerID string, sizeTobeFilled, bs int) error {
func fillDisk(targetPID int, sizeTobeFilled, bs int) error {

// Creating files to fill the required ephemeral storage size of block size of 4K
log.Infof("[Fill]: Filling ephemeral storage, size: %vKB", sizeTobeFilled)
dd := fmt.Sprintf("sudo dd if=/dev/urandom of=/diskfill/%v/diskfill bs=%vK count=%v", containerID, bs, strconv.Itoa(sizeTobeFilled/bs))
dd := fmt.Sprintf("sudo dd if=/dev/urandom of=/proc/%v/root/home/diskfill bs=%vK count=%v", targetPID, bs, strconv.Itoa(sizeTobeFilled/bs))
log.Infof("dd: {%v}", dd)
cmd := exec.Command("/bin/bash", "-c", dd)
_, err := cmd.CombinedOutput()
Expand Down Expand Up @@ -231,23 +236,23 @@ func getSizeToBeFilled(experimentsDetails *experimentTypes.ExperimentDetails, us
return needToBeFilled
}

// remedy will delete the target pod if target pod is evicted
// revertDiskFill will delete the target pod if target pod is evicted
// if target pod is still running then it will delete the files, which was created during chaos execution
func remedy(t targetDetails, clients clients.ClientSets) error {
func revertDiskFill(t targetDetails, clients clients.ClientSets) error {
pod, err := clients.KubeClient.CoreV1().Pods(t.Namespace).Get(context.Background(), t.Name, v1.GetOptions{})
if err != nil {
return err
}
// Deleting the pod as pod is already evicted
podReason := pod.Status.Reason
if podReason == "Evicted" {
// Deleting the pod as pod is already evicted
log.Warn("Target pod is evicted, deleting the pod")
if err := clients.KubeClient.CoreV1().Pods(t.Namespace).Delete(context.Background(), t.Name, v1.DeleteOptions{}); err != nil {
return err
}
} else {
// deleting the files after chaos execution
rm := fmt.Sprintf("sudo rm -rf /diskfill/%v/diskfill", t.ContainerId)
rm := fmt.Sprintf("sudo rm -rf /proc/%v/root/home/diskfill", t.TargetPID)
cmd := exec.Command("/bin/bash", "-c", rm)
out, err := cmd.CombinedOutput()
if err != nil {
Expand All @@ -259,7 +264,7 @@ func remedy(t targetDetails, clients clients.ClientSets) error {
return nil
}

//getENV fetches all the env variables from the runner pod
// getENV fetches all the env variables from the runner pod
func getENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ExperimentName = types.Getenv("EXPERIMENT_NAME", "")
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
Expand All @@ -271,6 +276,8 @@ func getENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.FillPercentage = types.Getenv("FILL_PERCENTAGE", "")
experimentDetails.EphemeralStorageMebibytes = types.Getenv("EPHEMERAL_STORAGE_MEBIBYTES", "")
experimentDetails.DataBlockSize, _ = strconv.Atoi(types.Getenv("DATA_BLOCK_SIZE", "256"))
experimentDetails.ContainerRuntime = types.Getenv("CONTAINER_RUNTIME", "")
experimentDetails.SocketPath = types.Getenv("SOCKET_PATH", "")
}

// abortWatcher continuously watch for the abort signals
Expand All @@ -284,7 +291,7 @@ func abortWatcher(targets []targetDetails, experimentsDetails *experimentTypes.E
retry := 3
for retry > 0 {
for _, t := range targets {
err := remedy(t, clients)
err := revertDiskFill(t, clients)
if err != nil {
log.Errorf("unable to kill disk-fill process, err :%v", err)
continue
Expand All @@ -301,7 +308,10 @@ func abortWatcher(targets []targetDetails, experimentsDetails *experimentTypes.E
}

func getDiskSizeToFill(t targetDetails, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets) (int, error) {
usedEphemeralStorageSize, err := getUsedEphemeralStorage(t.ContainerId)
usedEphemeralStorageSize, err := getUsedEphemeralStorage(t.TargetPID)
if err != nil {
return 0, err
}

// GetEphemeralStorageAttributes derive the ephemeral storage attributes from the target container
ephemeralStorageLimit, err := getEphemeralStorageAttributes(t, clients)
Expand All @@ -319,9 +329,9 @@ func getDiskSizeToFill(t targetDetails, experimentsDetails *experimentTypes.Expe
return sizeTobeFilled, nil
}

func getUsedEphemeralStorage(containerId string) (int, error) {
func getUsedEphemeralStorage(targetPID int) (int, error) {
// derive the used ephemeral storage size from the target container
du := fmt.Sprintf("sudo du /diskfill/%v", containerId)
du := fmt.Sprintf("sudo du /proc/%v/root", targetPID)
cmd := exec.Command("/bin/bash", "-c", du)
out, err := cmd.CombinedOutput()
if err != nil {
Expand All @@ -345,4 +355,5 @@ type targetDetails struct {
TargetContainer string
ContainerId string
SizeToFill int
TargetPID int
}
24 changes: 15 additions & 9 deletions chaoslib/litmus/disk-fill/lib/disk-fill.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

//PrepareDiskFill contains the prepration steps before chaos injection
// PrepareDiskFill contains the prepration steps before chaos injection
func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {

var err error
Expand Down Expand Up @@ -199,7 +199,7 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
// createHelperPod derive the attributes for helper pod and create the helper pod
func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, targets, appNodeName, runID string) error {

mountPropagationMode := apiv1.MountPropagationHostToContainer
privilegedEnable := true
terminationGracePeriodSeconds := int64(experimentsDetails.TerminationGracePeriodSeconds)

helperPod := &apiv1.Pod{
Expand All @@ -210,17 +210,19 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Annotations: chaosDetails.Annotations,
},
Spec: apiv1.PodSpec{
HostPID: true,
RestartPolicy: apiv1.RestartPolicyNever,
ImagePullSecrets: chaosDetails.ImagePullSecrets,
NodeName: appNodeName,
ServiceAccountName: experimentsDetails.ChaosServiceAccount,
TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,

Volumes: []apiv1.Volume{
{
Name: "udev",
Name: "socket-path",
VolumeSource: apiv1.VolumeSource{
HostPath: &apiv1.HostPathVolumeSource{
Path: experimentsDetails.ContainerPath,
Path: experimentsDetails.SocketPath,
},
},
},
Expand All @@ -241,11 +243,13 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Env: getPodEnv(experimentsDetails, targets),
VolumeMounts: []apiv1.VolumeMount{
{
Name: "udev",
MountPath: "/diskfill",
MountPropagation: &mountPropagationMode,
Name: "socket-path",
MountPath: experimentsDetails.SocketPath,
},
},
SecurityContext: &apiv1.SecurityContext{
Privileged: &privilegedEnable,
},
},
},
},
Expand All @@ -270,13 +274,15 @@ func getPodEnv(experimentsDetails *experimentTypes.ExperimentDetails, targets st
SetEnv("EPHEMERAL_STORAGE_MEBIBYTES", experimentsDetails.EphemeralStorageMebibytes).
SetEnv("DATA_BLOCK_SIZE", strconv.Itoa(experimentsDetails.DataBlockSize)).
SetEnv("INSTANCE_ID", experimentsDetails.InstanceID).
SetEnv("SOCKET_PATH", experimentsDetails.SocketPath).
SetEnv("CONTAINER_RUNTIME", experimentsDetails.ContainerRuntime).
SetEnvFromDownwardAPI("v1", "metadata.name")

return envDetails.ENV
}

//setChaosTunables will setup a random value within a given range of values
//If the value is not provided in range it'll setup the initial provided value.
// setChaosTunables will setup a random value within a given range of values
// If the value is not provided in range it'll setup the initial provided value.
func setChaosTunables(experimentsDetails *experimentTypes.ExperimentDetails) {
experimentsDetails.FillPercentage = common.ValidateRange(experimentsDetails.FillPercentage)
experimentsDetails.EphemeralStorageMebibytes = common.ValidateRange(experimentsDetails.EphemeralStorageMebibytes)
Expand Down
22 changes: 0 additions & 22 deletions experiments/generic/disk-fill/experiment/disk-fill.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,17 +83,6 @@ func DiskFill(clients clients.ClientSets) {
}
}

//PRE-CHAOS AUXILIARY APPLICATION STATUS CHECK
if experimentsDetails.AuxiliaryAppInfo != "" {
log.Info("[Status]: Verify that the Auxiliary Applications are running (pre-chaos)")
if err := status.CheckAuxiliaryApplicationStatus(experimentsDetails.AuxiliaryAppInfo, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Auxiliary Application status check failed, err: %v", err)
failStep := "[pre-chaos]: Failed to verify that the Auxiliary Applications are in running state, err: " + err.Error()
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
// marking AUT as running, as we already checked the status of application under test
msg := common.GetStatusMessage(chaosDetails.DefaultHealthCheck, "AUT: Running", "")
Expand Down Expand Up @@ -149,17 +138,6 @@ func DiskFill(clients clients.ClientSets) {
}
}

//POST-CHAOS AUXILIARY APPLICATION STATUS CHECK
if experimentsDetails.AuxiliaryAppInfo != "" {
log.Info("[Status]: Verify that the Auxiliary Applications are running (post-chaos)")
if err := status.CheckAuxiliaryApplicationStatus(experimentsDetails.AuxiliaryAppInfo, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Auxiliary Application status check failed, err: %v", err)
failStep := "[post-chaos]: Failed to verify that the Auxiliary Applications are running, err: " + err.Error()
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
// marking AUT as running, as we already checked the status of application under test
msg := common.GetStatusMessage(chaosDetails.DefaultHealthCheck, "AUT: Running", "")
Expand Down
6 changes: 3 additions & 3 deletions pkg/generic/disk-fill/environment/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/litmuschaos/litmus-go/pkg/types"
)

//GetENV fetches all the env variables from the runner pod
// GetENV fetches all the env variables from the runner pod
func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ExperimentName = types.Getenv("EXPERIMENT_NAME", "disk-fill")
experimentDetails.ChaosNamespace = types.Getenv("CHAOS_NAMESPACE", "litmus")
Expand All @@ -20,9 +20,9 @@ func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ChaosUID = clientTypes.UID(types.Getenv("CHAOS_UID", ""))
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
experimentDetails.ChaosPodName = types.Getenv("POD_NAME", "")
experimentDetails.AuxiliaryAppInfo = types.Getenv("AUXILIARY_APPINFO", "")
experimentDetails.TargetContainer = types.Getenv("TARGET_CONTAINER", "")
experimentDetails.ContainerPath = types.Getenv("CONTAINER_PATH", "/var/lib/docker/containers")
experimentDetails.ContainerRuntime = types.Getenv("CONTAINER_RUNTIME", "docker")
experimentDetails.SocketPath = types.Getenv("SOCKET_PATH", "/var/run/docker.sock")
experimentDetails.FillPercentage = types.Getenv("FILL_PERCENTAGE", "80")
experimentDetails.Delay, _ = strconv.Atoi(types.Getenv("STATUS_CHECK_DELAY", "2"))
experimentDetails.Timeout, _ = strconv.Atoi(types.Getenv("STATUS_CHECK_TIMEOUT", "180"))
Expand Down
4 changes: 2 additions & 2 deletions pkg/generic/disk-fill/types/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ type ExperimentDetails struct {
ChaosNamespace string
ChaosPodName string
TargetContainer string
AuxiliaryAppInfo string
FillPercentage string
ContainerPath string
ContainerRuntime string
SocketPath string
RunID string
Timeout int
Delay int
Expand Down