Skip to content

Commit

Permalink
changed dd to use nsenter
Browse files Browse the repository at this point in the history 
Signed-off-by: Akash Shrivastava <akash.shrivastava@harness.io>
  • Loading branch information
@avaakash
avaakash committed Nov 18, 2022
1 parent f16249f commit fdd7372
Show file tree
Hide file tree
Showing 5 changed files with 42 additions and 51 deletions.
35 changes: 21 additions & 14 deletions chaoslib/litmus/disk-fill/helper/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,17 +63,22 @@ func Helper(clients clients.ClientSets) {
}
}

//diskFill contains steps to inject disk-fill chaos
// diskFill contains steps to inject disk-fill chaos
func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails) error {

// Derive the container id of the target container
containerID, err := common.GetContainerID(experimentsDetails.AppNS, experimentsDetails.TargetPods, experimentsDetails.TargetContainer, clients)
if err != nil {
return err
}
// extract out the pid of the target container
targetPID, err := common.GetPID(experimentsDetails.ContainerRuntime, containerID, experimentsDetails.SocketPath)
if err != nil {
return err
}

// derive the used ephemeral storage size from the target container
du := fmt.Sprintf("sudo du /diskfill/%v", containerID)
du := fmt.Sprintf("sudo du /proc/%v/root", targetPID)
cmd := exec.Command("/bin/bash", "-c", du)
out, err := cmd.CombinedOutput()
if err != nil {
Expand Down Expand Up @@ -119,11 +124,11 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
}

// watching for the abort signal and revert the chaos
go abortWatcher(experimentsDetails, clients, containerID, resultDetails.Name)
go abortWatcher(experimentsDetails, clients, targetPID, resultDetails.Name)

if sizeTobeFilled > 0 {

if err := fillDisk(containerID, sizeTobeFilled, experimentsDetails.DataBlockSize); err != nil {
if err := fillDisk(targetPID, sizeTobeFilled, experimentsDetails.DataBlockSize); err != nil {
log.Error(string(out))
return err
}
Expand All @@ -140,7 +145,7 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli

// It will delete the target pod if target pod is evicted
// if target pod is still running then it will delete all the files, which was created earlier during chaos execution
err = remedy(experimentsDetails, clients, containerID)
err = revertDiskFill(experimentsDetails, clients, targetPID)
if err != nil {
return errors.Errorf("unable to perform remedy operation, err: %v", err)
}
Expand All @@ -154,7 +159,7 @@ func diskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients cli
}

// fillDisk fill the ephemeral disk by creating files
func fillDisk(containerID string, sizeTobeFilled, bs int) error {
func fillDisk(targetPID, sizeTobeFilled, bs int) error {

select {
case <-inject:
Expand All @@ -163,7 +168,7 @@ func fillDisk(containerID string, sizeTobeFilled, bs int) error {
default:
// Creating files to fill the required ephemeral storage size of block size of 4K
log.Infof("[Fill]: Filling ephemeral storage, size: %vKB", sizeTobeFilled)
dd := fmt.Sprintf("sudo dd if=/dev/urandom of=/diskfill/%v/diskfill bs=%vK count=%v", containerID, bs, strconv.Itoa(sizeTobeFilled/bs))
dd := fmt.Sprintf("sudo dd if=/dev/urandom of=/proc/%v/root/home/diskfill bs=%vK count=%v", targetPID, bs, strconv.Itoa(sizeTobeFilled/bs))
log.Infof("dd: {%v}", dd)
cmd := exec.Command("/bin/bash", "-c", dd)
_, err := cmd.CombinedOutput()
Expand Down Expand Up @@ -226,23 +231,23 @@ func getSizeToBeFilled(experimentsDetails *experimentTypes.ExperimentDetails, us
return needToBeFilled
}

// remedy will delete the target pod if target pod is evicted
// revertDiskFill will delete the target pod if target pod is evicted
// if target pod is still running then it will delete the files, which was created during chaos execution
func remedy(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, containerID string) error {
func revertDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, targetPID int) error {
pod, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.AppNS).Get(context.Background(), experimentsDetails.TargetPods, v1.GetOptions{})
if err != nil {
return err
}
// Deleting the pod as pod is already evicted
podReason := pod.Status.Reason
if podReason == "Evicted" {
// Deleting the pod as pod is already evicted
log.Warn("Target pod is evicted, deleting the pod")
if err := clients.KubeClient.CoreV1().Pods(experimentsDetails.AppNS).Delete(context.Background(), experimentsDetails.TargetPods, v1.DeleteOptions{}); err != nil {
return err
}
} else {
// deleting the files after chaos execution
rm := fmt.Sprintf("sudo rm -rf /diskfill/%v/diskfill", containerID)
rm := fmt.Sprintf("sudo rm -rf /proc/%v/root/home/diskfill", targetPID)
cmd := exec.Command("/bin/bash", "-c", rm)
out, err := cmd.CombinedOutput()
if err != nil {
Expand All @@ -253,7 +258,7 @@ func remedy(experimentsDetails *experimentTypes.ExperimentDetails, clients clien
return nil
}

//getENV fetches all the env variables from the runner pod
// getENV fetches all the env variables from the runner pod
func getENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ExperimentName = types.Getenv("EXPERIMENT_NAME", "")
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
Expand All @@ -268,10 +273,12 @@ func getENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.FillPercentage = types.Getenv("FILL_PERCENTAGE", "")
experimentDetails.EphemeralStorageMebibytes = types.Getenv("EPHEMERAL_STORAGE_MEBIBYTES", "")
experimentDetails.DataBlockSize, _ = strconv.Atoi(types.Getenv("DATA_BLOCK_SIZE", "256"))
experimentDetails.ContainerRuntime = types.Getenv("CONTAINER_RUNTIME", "")
experimentDetails.SocketPath = types.Getenv("SOCKET_PATH", "")
}

// abortWatcher continuously watch for the abort signals
func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, containerID, resultName string) {
func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, targetPID int, resultName string) {
// waiting till the abort signal received
<-abort

Expand All @@ -280,7 +287,7 @@ func abortWatcher(experimentsDetails *experimentTypes.ExperimentDetails, clients
// retry thrice for the chaos revert
retry := 3
for retry > 0 {
if err := remedy(experimentsDetails, clients, containerID); err != nil {
if err := revertDiskFill(experimentsDetails, clients, targetPID); err != nil {
log.Errorf("unable to perform remedy operation, err: %v", err)
}
retry--
Expand Down
26 changes: 16 additions & 10 deletions chaoslib/litmus/disk-fill/lib/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

//PrepareDiskFill contains the prepration steps before chaos injection
// PrepareDiskFill contains the prepration steps before chaos injection
func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {

targetPodList := apiv1.PodList{}
Expand Down Expand Up @@ -67,7 +67,7 @@ func PrepareDiskFill(experimentsDetails *experimentTypes.ExperimentDetails, clie
for _, pod := range targetPodList.Items {
podNames = append(podNames, pod.Name)
}
log.Infof("Target pods list for chaos, %v", podNames)
log.Infof("Target pods list for chaos: %v", podNames)

//Waiting for the ramp time before chaos injection
if experimentsDetails.RampTime != 0 {
Expand Down Expand Up @@ -227,7 +227,7 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
// createHelperPod derive the attributes for helper pod and create the helper pod
func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, chaosDetails *types.ChaosDetails, appName, appNodeName, runID, labelSuffix string) error {

mountPropagationMode := apiv1.MountPropagationHostToContainer
privilegedEnable := true
terminationGracePeriodSeconds := int64(experimentsDetails.TerminationGracePeriodSeconds)

helperPod := &apiv1.Pod{
Expand All @@ -238,17 +238,19 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Annotations: chaosDetails.Annotations,
},
Spec: apiv1.PodSpec{
HostPID: true,
RestartPolicy: apiv1.RestartPolicyNever,
ImagePullSecrets: chaosDetails.ImagePullSecrets,
NodeName: appNodeName,
ServiceAccountName: experimentsDetails.ChaosServiceAccount,
TerminationGracePeriodSeconds: &terminationGracePeriodSeconds,

Volumes: []apiv1.Volume{
{
Name: "udev",
Name: "socket-path",
VolumeSource: apiv1.VolumeSource{
HostPath: &apiv1.HostPathVolumeSource{
Path: experimentsDetails.ContainerPath,
Path: experimentsDetails.SocketPath,
},
},
},
Expand All @@ -269,11 +271,13 @@ func createHelperPod(experimentsDetails *experimentTypes.ExperimentDetails, clie
Env: getPodEnv(experimentsDetails, appName),
VolumeMounts: []apiv1.VolumeMount{
{
Name: "udev",
MountPath: "/diskfill",
MountPropagation: &mountPropagationMode,
Name: "socket-path",
MountPath: experimentsDetails.SocketPath,
},
},
SecurityContext: &apiv1.SecurityContext{
Privileged: &privilegedEnable,
},
},
},
},
Expand All @@ -299,13 +303,15 @@ func getPodEnv(experimentsDetails *experimentTypes.ExperimentDetails, podName st
SetEnv("EPHEMERAL_STORAGE_MEBIBYTES", experimentsDetails.EphemeralStorageMebibytes).
SetEnv("DATA_BLOCK_SIZE", strconv.Itoa(experimentsDetails.DataBlockSize)).
SetEnv("INSTANCE_ID", experimentsDetails.InstanceID).
SetEnv("SOCKET_PATH", experimentsDetails.SocketPath).
SetEnv("CONTAINER_RUNTIME", experimentsDetails.ContainerRuntime).
SetEnvFromDownwardAPI("v1", "metadata.name")

return envDetails.ENV
}

//setChaosTunables will setup a random value within a given range of values
//If the value is not provided in range it'll setup the initial provided value.
// setChaosTunables will setup a random value within a given range of values
// If the value is not provided in range it'll setup the initial provided value.
func setChaosTunables(experimentsDetails *experimentTypes.ExperimentDetails) {
experimentsDetails.FillPercentage = common.ValidateRange(experimentsDetails.FillPercentage)
experimentsDetails.EphemeralStorageMebibytes = common.ValidateRange(experimentsDetails.EphemeralStorageMebibytes)
Expand Down
22 changes: 0 additions & 22 deletions experiments/generic/disk-fill/experiment/disk-fill.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,17 +85,6 @@ func DiskFill(clients clients.ClientSets) {
}
}

//PRE-CHAOS AUXILIARY APPLICATION STATUS CHECK
if experimentsDetails.AuxiliaryAppInfo != "" {
log.Info("[Status]: Verify that the Auxiliary Applications are running (pre-chaos)")
if err := status.CheckAuxiliaryApplicationStatus(experimentsDetails.AuxiliaryAppInfo, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Auxiliary Application status check failed, err: %v", err)
failStep := "[pre-chaos]: Failed to verify that the Auxiliary Applications are in running state, err: " + err.Error()
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
// marking AUT as running, as we already checked the status of application under test
msg := common.GetStatusMessage(chaosDetails.DefaultHealthCheck, "AUT: Running", "")
Expand Down Expand Up @@ -151,17 +140,6 @@ func DiskFill(clients clients.ClientSets) {
}
}

//POST-CHAOS AUXILIARY APPLICATION STATUS CHECK
if experimentsDetails.AuxiliaryAppInfo != "" {
log.Info("[Status]: Verify that the Auxiliary Applications are running (post-chaos)")
if err := status.CheckAuxiliaryApplicationStatus(experimentsDetails.AuxiliaryAppInfo, experimentsDetails.Timeout, experimentsDetails.Delay, clients); err != nil {
log.Errorf("Auxiliary Application status check failed, err: %v", err)
failStep := "[post-chaos]: Failed to verify that the Auxiliary Applications are running, err: " + err.Error()
result.RecordAfterFailure(&chaosDetails, &resultDetails, failStep, clients, &eventsDetails)
return
}
}

if experimentsDetails.EngineName != "" {
// marking AUT as running, as we already checked the status of application under test
msg := common.GetStatusMessage(chaosDetails.DefaultHealthCheck, "AUT: Running", "")
Expand Down
6 changes: 3 additions & 3 deletions pkg/generic/disk-fill/environment/environment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import (
"github.com/litmuschaos/litmus-go/pkg/types"
)

//GetENV fetches all the env variables from the runner pod
// GetENV fetches all the env variables from the runner pod
func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ExperimentName = types.Getenv("EXPERIMENT_NAME", "disk-fill")
experimentDetails.ChaosNamespace = types.Getenv("CHAOS_NAMESPACE", "litmus")
Expand All @@ -23,9 +23,9 @@ func GetENV(experimentDetails *experimentTypes.ExperimentDetails) {
experimentDetails.ChaosUID = clientTypes.UID(types.Getenv("CHAOS_UID", ""))
experimentDetails.InstanceID = types.Getenv("INSTANCE_ID", "")
experimentDetails.ChaosPodName = types.Getenv("POD_NAME", "")
experimentDetails.AuxiliaryAppInfo = types.Getenv("AUXILIARY_APPINFO", "")
experimentDetails.TargetContainer = types.Getenv("TARGET_CONTAINER", "")
experimentDetails.ContainerPath = types.Getenv("CONTAINER_PATH", "/var/lib/docker/containers")
experimentDetails.ContainerRuntime = types.Getenv("CONTAINER_RUNTIME", "docker")
experimentDetails.SocketPath = types.Getenv("SOCKET_PATH", "/var/lib/docker")
experimentDetails.FillPercentage = types.Getenv("FILL_PERCENTAGE", "80")
experimentDetails.Delay, _ = strconv.Atoi(types.Getenv("STATUS_CHECK_DELAY", "2"))
experimentDetails.Timeout, _ = strconv.Atoi(types.Getenv("STATUS_CHECK_TIMEOUT", "180"))
Expand Down
4 changes: 2 additions & 2 deletions pkg/generic/disk-fill/types/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,9 @@ type ExperimentDetails struct {
ChaosNamespace string
ChaosPodName string
TargetContainer string
AuxiliaryAppInfo string
FillPercentage string
ContainerPath string
ContainerRuntime string
SocketPath string
RunID string
Timeout int
Delay int
Expand Down

0 comments on commit fdd7372

Please sign in to comment.