Skip to content

Commit

Permalink
changed dd to use nsenter
Browse files Browse the repository at this point in the history 
Signed-off-by: Akash Shrivastava <akash.shrivastava@harness.io>
  • Loading branch information
@avaakash
avaakash committed Nov 22, 2022
1 parent 0e18911 commit 4802d58
Show file tree
Hide file tree
Showing 5 changed files with 51 additions and 56 deletions.
51 changes: 31 additions & 20 deletions chaoslib/litmus/disk-fill/helper/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,11 @@ import (

var inject, abort chan os.Signal

var err error

// Helper injects the disk-fill chaos
func Helper(clients clients.ClientSets) {

var err error

experimentsDetails := experimentTypes.ExperimentDetails{}
eventsDetails := types.EventDetails{}
chaosDetails := types.ChaosDetails{}
Expand Down Expand Up @@ -60,12 +60,12 @@ func Helper(clients clients.ClientSets) {
// Set the chaos result uid
result.SetResultUID(&resultDetails, clients, &chaosDetails)

if err := diskFill(&experimentsDetails, clients, &eventsDetails, &chaosDetails, &resultDetails); err != nil {
if err = diskFill(&experimentsDetails, clients, &eventsDetails, &chaosDetails, &resultDetails); err != nil {
log.Fatalf("helper pod failed, err: %v", err)
}
}

//diskFill contains steps to inject disk-fill chaos
// diskFill contains steps to inject disk-fill chaos
func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails) error {

targetList, err := common.ParseTargets()
Expand All @@ -88,6 +88,12 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
return err
}

// extract out the pid of the target container
td.TargetPID, err = common.GetPID(experimentsDetails.ContainerRuntime, td.ContainerId, experimentsDetails.SocketPath)
if err != nil {
return err
}

td.SizeToFill, err = getDiskSizeToFill(td, experimentsDetails, clients)
if err != nil {
return err
Expand Down Expand Up @@ -119,16 +125,15 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
os.Exit(1)
default:
}

for _, t := range targets {
if t.SizeToFill > 0 {
if err := fillDisk(t.ContainerId, t.SizeToFill, experimentsDetails.DataBlockSize); err != nil {
if err = fillDisk(t.TargetPID, t.SizeToFill, experimentsDetails.DataBlockSize); err != nil {
return err
}
log.Infof("successfully injected chaos on target: {name: %s, namespace: %v, container: %v}", t.Name, t.Namespace, t.TargetContainer)
if err = result.AnnotateChaosResult(resultDetails.Name, chaosDetails.ChaosNamespace, "injected", "pod", t.Name); err != nil {
if remedyErr := remedy(t, clients); remedyErr != nil {
return remedyErr
if revertErr := revertDiskFill(t, clients); revertErr != nil {
return revertErr
}
return err
}
Expand All @@ -148,7 +153,7 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
for _, t := range targets {
// It will delete the target pod if target pod is evicted
// if target pod is still running then it will delete all the files, which was created earlier during chaos execution
err = remedy(t, clients)
err = revertDiskFill(t, clients)
if err != nil {
errList = append(errList, err.Error())
continue
Expand All @@ -166,11 +171,11 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
}

// fillDisk fill the ephemeral disk by creating files
func fillDisk(containerID string, sizeTobeFilled, bs int) error {
func fillDisk(targetPID int, sizeTobeFilled, bs int) error {

// Creating files to fill the required ephemeral storage size of block size of 4K
log.Infof("[Fill]: Filling ephemeral storage, size: %vKB", sizeTobeFilled)
dd := fmt.Sprintf("sudo dd if=/dev/urandom of=/diskfill/%v/diskfill bs=%vK count=%v", containerID, bs, strconv.Itoa(sizeTobeFilled/bs))
dd := fmt.Sprintf("sudo dd if=/dev/urandom of=/proc/%v/root/home/diskfill bs=%vK count=%v", targetPID, bs, strconv.Itoa(sizeTobeFilled/bs))
log.Infof("dd: {%v}", dd)
cmd := exec.Command("/bin/bash", "-c", dd)
_, err := cmd.CombinedOutput()
Expand Down Expand Up @@ -231,23 +236,23 @@ func getSizeToBeFilled(experimentsDetails *experimentTypes.ExperimentDetails, us
return needToBeFilled
}

// remedy will delete the target pod if target pod is evicted
// revertDiskFill will delete the target pod if target pod is evicted
// if target pod is still running then it will delete the files, which was created during chaos execution
func remedy(t targetDetails, clients clients.ClientSets) error {
func revertDiskFill(t targetDetails, clients clients.ClientSets) error {
pod, err := clients.KubeClient.CoreV1().Pods(t.Namespace).Get(context.Background(), t.Name, v1.GetOptions{})
if err != nil {
return err
}
// Deleting the pod as pod is already evicted
podReason := pod.Status.Reason
if podReason == "Evicted" {
// Deleting the pod as pod is already evicted
log.Warn("Target pod is evicted, deleting the pod")
if err := clients.KubeClient.CoreV1().Pods(t.Namespace).Delete(context.Background(), t.Name, v1.DeleteOptions{}); err != nil {
return err
}
} else {
// deleting the files after chaos execution
rm := fmt.Sprintf("sudo rm -rf /diskfill/%v/diskfill", t.ContainerId)
rm := fmt.Sprintf("sudo rm -rf /proc/%v/root/home/diskfill", t.TargetPID)
cmd := exec.Command("/bin/bash", "-c", rm)
out, err := cmd.CombinedOutput()
if err != nil {
Expand All @@ -259,7 +264,7 @@ func remedy(t targetDetails, clients clients.ClientSets) error {
return nil
}

//getENV fetches all the env variables from the runner pod
// getENV fetches all the env variables from the runner pod
func getENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ExperimentName = types.Getenv("EXPERIMENT_NAME", "")
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
Expand All @@ -271,6 +276,8 @@ func getENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.FillPercentage = types.Getenv("FILL_PERCENTAGE", "")
experimentDetails.EphemeralStorageMebibytes = types.Getenv("EPHEMERAL_STORAGE_MEBIBYTES", "")
experimentDetails.DataBlockSize, _ = strconv.Atoi(types.Getenv("DATA_BLOCK_SIZE", "256"))
experimentDetails.ContainerRuntime = types.Getenv("CONTAINER_RUNTIME", "")
experimentDetails.SocketPath = types.Getenv("SOCKET_PATH", "")
}

// abortWatcher continuously watch for the abort signals
Expand All @@ -284,7 +291,7 @@ func abortWatcher(targets []targetDetails, experimentsDetails *experimentTypes.E
retry := 3
for retry > 0 {
for _, t := range targets {
err := remedy(t, clients)
err := revertDiskFill(t, clients)
if err != nil {
log.Errorf("unable to kill disk-fill process, err :%v", err)
continue
Expand All @@ -301,7 +308,10 @@ func abortWatcher(targets []targetDetails, experimentsDetails *experimentTypes.E
}

func getDiskSizeToFill(t targetDetails, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets) (int, error) {
usedEphemeralStorageSize, err := getUsedEphemeralStorage(t.ContainerId)
usedEphemeralStorageSize, err := getUsedEphemeralStorage(t.TargetPID)
if err != nil {
return 0, err
}

// GetEphemeralStorageAttributes derive the ephemeral storage attributes from the target container
ephemeralStorageLimit, err := getEphemeralStorageAttributes(t, clients)
Expand All @@ -319,9 +329,9 @@ func getDiskSizeToFill(t targetDetails, experimentsDetails *experimentTypes.Expe
return sizeTobeFilled, nil
}

func getUsedEphemeralStorage(containerId string) (int, error) {
func getUsedEphemeralStorage(targetPID int) (int, error) {
// derive the used ephemeral storage size from the target container
du := fmt.Sprintf("sudo du /diskfill/%v", containerId)
du := fmt.Sprintf("sudo du /proc/%v/root", targetPID)
cmd := exec.Command("/bin/bash", "-c", du)
out, err := cmd.CombinedOutput()
if err != nil {
Expand All @@ -345,4 +355,5 @@ type targetDetails struct {
TargetContainer string
ContainerId string
SizeToFill int
TargetPID int
}
24 changes: 15 additions & 9 deletions chaoslib/litmus/disk-fill/lib/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

//PrepareDiskFill contains the prepration steps before chaos injection
// PrepareDiskFill contains the prepration steps before chaos injection
func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {

var err error
Expand Down Expand Up @@ -199,7 +199,7 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
// createHelperPod derive the attributes for helper pod and create the helper pod
func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, targets, appNodeName, runID string) error {

mountPropagationMode := apiv1.MountPropagationHostToContainer
privilegedEnable := true
terminationGracePeriodSeconds := int64(experimentsDetails.TerminationGracePeriodSeconds)

helperPod := &apiv1.Pod{
Expand All @@ -210,17 +210,19 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Annotations: chaosDetails.Annotations,
},
Spec: apiv1.PodSpec{
HostPID: true,
RestartPolicy: apiv1.RestartPolicyNever,
ImagePullSecrets: chaosDetails.ImagePullSecrets,
NodeName: appNodeName,
ServiceAccountName: experimentsDetails.ChaosServiceAccount,
TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,

Volumes: []apiv1.Volume{
{
Name: "udev",
Name: "socket-path",
VolumeSource: apiv1.VolumeSource{
HostPath: &apiv1.HostPathVolumeSource{
Path: experimentsDetails.ContainerPath,
Path: experimentsDetails.SocketPath,
},
},
},
Expand All @@ -241,11 +243,13 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Env: getPodEnv(experimentsDetails, targets),
VolumeMounts: []apiv1.VolumeMount{
{
Name: "udev",
MountPath: "/diskfill",
MountPropagation: &mountPropagationMode,
Name: "socket-path",
MountPath: experimentsDetails.SocketPath,
},
},
SecurityContext: &apiv1.SecurityContext{
Privileged: &privilegedEnable,
},
},
},
},
Expand All @@ -270,13 +274,15 @@ func getPodEnv(experimentsDetails *experimentTypes.ExperimentDetails, targets st
SetEnv("EPHEMERAL_STORAGE_MEBIBYTES", experimentsDetails.EphemeralStorageMebibytes).
SetEnv("DATA_BLOCK_SIZE", strconv.Itoa(experimentsDetails.DataBlockSize)).
SetEnv("INSTANCE_ID", experimentsDetails.InstanceID).
SetEnv("SOCKET_PATH", experimentsDetails.SocketPath).
SetEnv("CONTAINER_RUNTIME", experimentsDetails.ContainerRuntime).
SetEnvFromDownwardAPI("v1", "metadata.name")

return envDetails.ENV
}

//setChaosTunables will setup a random value within a given range of values
//If the value is not provided in range it'll setup the initial provided value.
// setChaosTunables will setup a random value within a given range of values
// If the value is not provided in range it'll setup the initial provided value.
func setChaosTunables(experimentsDetails *experimentTypes.ExperimentDetails) {
experimentsDetails.FillPercentage = common.ValidateRange(experimentsDetails.FillPercentage)
experimentsDetails.EphemeralStorageMebibytes = common.ValidateRange(experimentsDetails.EphemeralStorageMebibytes)
Expand Down
22 changes: 0 additions & 22 deletions experiments/generic/disk-fill/experiment/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,17 +83,6 @@ func DiskFill(clients clients.ClientSets) {
}
}

//PRE-CHAOS AUXILIARY APPLICATION STATUS CHECK
if experimentsDetails.AuxiliaryAppInfo != "" {
log.Info("[Status]: Verify that the Auxiliary Applications are running (pre-chaos)")
if err := status.CheckAuxiliaryApplicationStatus(experimentsDetails.AuxiliaryAppInfo, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Auxiliary Application status check failed, err: %v", err)
failStep := "[pre-chaos]: Failed to verify that the Auxiliary Applications are in running state, err: " + err.Error()
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
// marking AUT as running, as we already checked the status of application under test
msg := common.GetStatusMessage(chaosDetails.DefaultHealthCheck, "AUT: Running", "")
Expand Down Expand Up @@ -149,17 +138,6 @@ func DiskFill(clients clients.ClientSets) {
}
}

//POST-CHAOS AUXILIARY APPLICATION STATUS CHECK
if experimentsDetails.AuxiliaryAppInfo != "" {
log.Info("[Status]: Verify that the Auxiliary Applications are running (post-chaos)")
if err := status.CheckAuxiliaryApplicationStatus(experimentsDetails.AuxiliaryAppInfo, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Auxiliary Application status check failed, err: %v", err)
failStep := "[post-chaos]: Failed to verify that the Auxiliary Applications are running, err: " + err.Error()
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
// marking AUT as running, as we already checked the status of application under test
msg := common.GetStatusMessage(chaosDetails.DefaultHealthCheck, "AUT: Running", "")
Expand Down
6 changes: 3 additions & 3 deletions pkg/generic/disk-fill/environment/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/litmuschaos/litmus-go/pkg/types"
)

//GetENV fetches all the env variables from the runner pod
// GetENV fetches all the env variables from the runner pod
func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ExperimentName = types.Getenv("EXPERIMENT_NAME", "disk-fill")
experimentDetails.ChaosNamespace = types.Getenv("CHAOS_NAMESPACE", "litmus")
Expand All @@ -20,9 +20,9 @@ func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ChaosUID = clientTypes.UID(types.Getenv("CHAOS_UID", ""))
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
experimentDetails.ChaosPodName = types.Getenv("POD_NAME", "")
experimentDetails.AuxiliaryAppInfo = types.Getenv("AUXILIARY_APPINFO", "")
experimentDetails.TargetContainer = types.Getenv("TARGET_CONTAINER", "")
experimentDetails.ContainerPath = types.Getenv("CONTAINER_PATH", "/var/lib/docker/containers")
experimentDetails.ContainerRuntime = types.Getenv("CONTAINER_RUNTIME", "docker")
experimentDetails.SocketPath = types.Getenv("SOCKET_PATH", "/var/run/docker.sock")
experimentDetails.FillPercentage = types.Getenv("FILL_PERCENTAGE", "80")
experimentDetails.Delay, _ = strconv.Atoi(types.Getenv("STATUS_CHECK_DELAY", "2"))
experimentDetails.Timeout, _ = strconv.Atoi(types.Getenv("STATUS_CHECK_TIMEOUT", "180"))
Expand Down
4 changes: 2 additions & 2 deletions pkg/generic/disk-fill/types/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ type ExperimentDetails struct {
ChaosNamespace string
ChaosPodName string
TargetContainer string
AuxiliaryAppInfo string
FillPercentage string
ContainerPath string
ContainerRuntime string
SocketPath string
RunID string
Timeout int
Delay int
Expand Down

0 comments on commit 4802d58

Please sign in to comment.