Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(pod-delete): Adding target details inside chaosresult #336

Merged
merged 7 commits into from
Jun 11, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions chaoslib/litmus/pod-autoscaler/lib/pod-autoscaler.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func PreparePodAutoscaler(experimentsDetails *experimentTypes.ExperimentDetails,
return errors.Errorf("fail to perform autoscaling, err: %v", err)
}

if err = autoscalerRecoveryInDeployment(experimentsDetails, clients, appsUnderTest); err != nil {
if err = autoscalerRecoveryInDeployment(experimentsDetails, clients, appsUnderTest, chaosDetails); err != nil {
return errors.Errorf("fail to rollback the autoscaling, err: %v", err)
}

Expand All @@ -93,7 +93,7 @@ func PreparePodAutoscaler(experimentsDetails *experimentTypes.ExperimentDetails,
return errors.Errorf("fail to perform autoscaling, err: %v", err)
}

if err = autoscalerRecoveryInStatefulset(experimentsDetails, clients, appsUnderTest); err != nil {
if err = autoscalerRecoveryInStatefulset(experimentsDetails, clients, appsUnderTest, chaosDetails); err != nil {
return errors.Errorf("fail to rollback the autoscaling, err: %v", err)
}

Expand Down Expand Up @@ -171,6 +171,7 @@ func podAutoscalerChaosInDeployment(experimentsDetails *experimentTypes.Experime
if err != nil {
return err
}
common.SetTargets(app.AppName, "injected", "deployment", chaosDetails)
}
return nil
})
Expand Down Expand Up @@ -204,6 +205,7 @@ func podAutoscalerChaosInStatefulset(experimentsDetails *experimentTypes.Experim
if err != nil {
return err
}
common.SetTargets(app.AppName, "injected", "statefulset", chaosDetails)
}
return nil
})
Expand Down Expand Up @@ -245,7 +247,7 @@ func deploymentStatusCheck(experimentsDetails *experimentTypes.ExperimentDetails
})

if isFailed {
if err = autoscalerRecoveryInDeployment(experimentsDetails, clients, appsUnderTest); err != nil {
if err = autoscalerRecoveryInDeployment(experimentsDetails, clients, appsUnderTest, chaosDetails); err != nil {
return errors.Errorf("fail to perform the autoscaler recovery of the deployment, err: %v", err)
}
return errors.Errorf("fail to scale the deployment to the desired replica count in the given chaos duration")
Expand Down Expand Up @@ -295,7 +297,7 @@ func statefulsetStatusCheck(experimentsDetails *experimentTypes.ExperimentDetail
})

if isFailed {
if err = autoscalerRecoveryInStatefulset(experimentsDetails, clients, appsUnderTest); err != nil {
if err = autoscalerRecoveryInStatefulset(experimentsDetails, clients, appsUnderTest, chaosDetails); err != nil {
return errors.Errorf("fail to perform the autoscaler recovery of the application, err: %v", err)
}
return errors.Errorf("fail to scale the application to the desired replica count in the given chaos duration")
Expand All @@ -321,7 +323,7 @@ func statefulsetStatusCheck(experimentsDetails *experimentTypes.ExperimentDetail
}

//autoscalerRecoveryInDeployment rollback the replicas to initial values in deployment
func autoscalerRecoveryInDeployment(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, appsUnderTest []experimentTypes.ApplicationUnderTest) error {
func autoscalerRecoveryInDeployment(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, appsUnderTest []experimentTypes.ApplicationUnderTest, chaosDetails *types.ChaosDetails) error {

// Scale back to initial number of replicas
retryErr := retries.RetryOnConflict(retries.DefaultRetry, func() error {
Expand All @@ -338,6 +340,7 @@ func autoscalerRecoveryInDeployment(experimentsDetails *experimentTypes.Experime
if err != nil {
return err
}
common.SetTargets(app.AppName, "reverted", "deployment", chaosDetails)
}
return nil
})
Expand Down Expand Up @@ -366,7 +369,7 @@ func autoscalerRecoveryInDeployment(experimentsDetails *experimentTypes.Experime
}

//autoscalerRecoveryInStatefulset rollback the replicas to initial values in deployment
func autoscalerRecoveryInStatefulset(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, appsUnderTest []experimentTypes.ApplicationUnderTest) error {
func autoscalerRecoveryInStatefulset(experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, appsUnderTest []experimentTypes.ApplicationUnderTest, chaosDetails *types.ChaosDetails) error {

// Scale back to initial number of replicas
retryErr := retries.RetryOnConflict(retries.DefaultRetry, func() error {
Expand All @@ -383,6 +386,7 @@ func autoscalerRecoveryInStatefulset(experimentsDetails *experimentTypes.Experim
if err != nil {
return err
}
common.SetTargets(app.AppName, "reverted", "statefulset", chaosDetails)
}
return nil
})
Expand Down Expand Up @@ -434,12 +438,12 @@ func abortPodAutoScalerChaos(appsUnderTest []experimentTypes.ApplicationUnderTes
// Other experiments have simpler "recoveries" that are more or less guaranteed to work.
switch strings.ToLower(experimentsDetails.AppKind) {
case "deployment", "deployments":
if err := autoscalerRecoveryInDeployment(experimentsDetails, clients, appsUnderTest); err != nil {
if err := autoscalerRecoveryInDeployment(experimentsDetails, clients, appsUnderTest, chaosDetails); err != nil {
log.Errorf("the recovery after abortion failed err: %v", err)
}

case "statefulset", "statefulsets":
if err := autoscalerRecoveryInStatefulset(experimentsDetails, clients, appsUnderTest); err != nil {
if err := autoscalerRecoveryInStatefulset(experimentsDetails, clients, appsUnderTest, chaosDetails); err != nil {
log.Errorf("the recovery after abortion failed err: %v", err)
}

Expand Down
12 changes: 10 additions & 2 deletions chaoslib/litmus/pod-delete/lib/pod-delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,11 @@ func injectChaosInSerialMode(experimentsDetails *experimentTypes.ExperimentDetai
for _, pod := range targetPodList.Items {
podNames = append(podNames, pod.Name)
}
log.Infof("Target pods list, %v", podNames)
log.Infof("Target pods list: %v", podNames)

for _, target := range chaosDetails.ParentsResources {
common.SetTargets(target, "targeted", chaosDetails.AppDetail.Kind, chaosDetails)
}

if experimentsDetails.EngineName != "" {
msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on application pod"
Expand Down Expand Up @@ -161,7 +165,11 @@ func injectChaosInParallelMode(experimentsDetails *experimentTypes.ExperimentDet
for _, pod := range targetPodList.Items {
podNames = append(podNames, pod.Name)
}
log.Infof("Target pods list for chaos, %v", podNames)
log.Infof("Target pods list: %v", podNames)

for _, target := range chaosDetails.ParentsResources {
common.SetTargets(target, "targeted", chaosDetails.AppDetail.Kind, chaosDetails)
}

if experimentsDetails.EngineName != "" {
msg := "Injecting " + experimentsDetails.ExperimentName + " chaos on application pod"
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ require (
github.com/imdario/mergo v0.3.9 // indirect
github.com/kr/pretty v0.2.0 // indirect
github.com/kyokomi/emoji v2.2.4+incompatible
github.com/litmuschaos/chaos-operator v0.0.0-20210609043500-9ee0e013933f
github.com/litmuschaos/chaos-operator v0.0.0-20210610071657-a58dbd939e73
github.com/mailru/easyjson v0.7.1 // indirect
github.com/openebs/maya v1.12.1
github.com/pkg/errors v0.9.1
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -511,8 +511,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
github.com/libopenstorage/openstorage v1.0.0/go.mod h1:Sp1sIObHjat1BeXhfMqLZ14wnOzEhNx2YQedreMcUyc=
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/litmuschaos/chaos-operator v0.0.0-20210609043500-9ee0e013933f h1:v6RR5cJmDAyQtabmKGQxlP6IFI5+X8LeSxeWUmePSgg=
github.com/litmuschaos/chaos-operator v0.0.0-20210609043500-9ee0e013933f/go.mod h1:QMjfAVIfwcpj/P1jikyz5+C5vWICiUXsFZMR7Ihnzro=
github.com/litmuschaos/chaos-operator v0.0.0-20210610071657-a58dbd939e73 h1:waXMmW5R0irh0gTSCMmXfYvEZ5cwaeMGulAwKp0zpFU=
github.com/litmuschaos/chaos-operator v0.0.0-20210610071657-a58dbd939e73/go.mod h1:QMjfAVIfwcpj/P1jikyz5+C5vWICiUXsFZMR7Ihnzro=
github.com/litmuschaos/elves v0.0.0-20201107015738-552d74669e3c/go.mod h1:DsbHGNUq/78NZozWVVI9Q6eBei4I+JjlkkD5aibJ3MQ=
github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
github.com/lpabon/godbc v0.1.1/go.mod h1:Jo9QV0cf3U6jZABgiJ2skINAXb9j8m51r07g4KI92ZA=
Expand Down Expand Up @@ -777,6 +777,7 @@ github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOV
github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
github.com/vmware/govmomi v0.20.1/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU=
Expand All @@ -798,6 +799,7 @@ github.com/yvasiyarov/gorelic v0.0.6/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96Tg
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
Expand Down
1 change: 1 addition & 0 deletions pkg/cassandra/pod-delete/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,4 +65,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, cassandraDetails
chaosDetails.AppDetail = appDetails
chaosDetails.ProbeImagePullPolicy = cassandraDetails.ChaoslibDetail.LIBImagePullPolicy
chaosDetails.Randomness, _ = strconv.ParseBool(common.Getenv("RANDOMNESS", "false"))
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/generic/container-kill/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.AppDetail = appDetails
chaosDetails.JobCleanupPolicy = common.Getenv("JOB_CLEANUP_POLICY", "retain")
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/generic/disk-fill/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,4 +61,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.AppDetail = appDetails
chaosDetails.JobCleanupPolicy = common.Getenv("JOB_CLEANUP_POLICY", "retain")
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/generic/network-chaos/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.AppDetail = appDetails
chaosDetails.JobCleanupPolicy = common.Getenv("JOB_CLEANUP_POLICY", "retain")
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
2 changes: 2 additions & 0 deletions pkg/generic/pod-autoscaler/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package environment
import (
"strconv"

"github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1"
experimentTypes "github.com/litmuschaos/litmus-go/pkg/generic/pod-autoscaler/types"
"github.com/litmuschaos/litmus-go/pkg/types"
"github.com/litmuschaos/litmus-go/pkg/utils/common"
Expand Down Expand Up @@ -49,4 +50,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.InstanceID = experimentDetails.InstanceID
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.AppDetail = appDetails
chaosDetails.Targets = []v1alpha1.TargetDetails{}
}
1 change: 1 addition & 0 deletions pkg/generic/pod-cpu-hog/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.Delay = experimentDetails.Delay
chaosDetails.AppDetail = appDetails
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
3 changes: 3 additions & 0 deletions pkg/generic/pod-delete/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package environment
import (
"strconv"

"github.com/litmuschaos/chaos-operator/pkg/apis/litmuschaos/v1alpha1"
clientTypes "k8s.io/apimachinery/pkg/types"

experimentTypes "github.com/litmuschaos/litmus-go/pkg/generic/pod-delete/types"
Expand Down Expand Up @@ -56,4 +57,6 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.AppDetail = appDetails
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.Randomness, _ = strconv.ParseBool(common.Getenv("RANDOMNESS", "false"))
chaosDetails.Targets = []v1alpha1.TargetDetails{}
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/generic/pod-dns-chaos/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,4 +79,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.AppDetail = appDetails
chaosDetails.JobCleanupPolicy = common.Getenv("JOB_CLEANUP_POLICY", "retain")
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/generic/pod-io-stress/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,4 +59,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.AppDetail = appDetails
chaosDetails.JobCleanupPolicy = common.Getenv("JOB_CLEANUP_POLICY", "retain")
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/generic/pod-memory-hog/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,4 +58,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, experimentDetail
chaosDetails.Delay = experimentDetails.Delay
chaosDetails.AppDetail = appDetails
chaosDetails.ProbeImagePullPolicy = experimentDetails.LIBImagePullPolicy
chaosDetails.ParentsResources = []string{}
}
1 change: 1 addition & 0 deletions pkg/kafka/environment/environment.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,4 +78,5 @@ func InitialiseChaosVariables(chaosDetails *types.ChaosDetails, kafkaDetails *ka
chaosDetails.AppDetail = appDetails
chaosDetails.ProbeImagePullPolicy = kafkaDetails.ChaoslibDetail.LIBImagePullPolicy
chaosDetails.Randomness, _ = strconv.ParseBool(common.Getenv("RANDOMNESS", "false"))
chaosDetails.ParentsResources = []string{}
}
3 changes: 3 additions & 0 deletions pkg/result/chaosresult.go
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ func InitializeChaosResult(chaosDetails *types.ChaosDetails, clients clients.Cli
PassedRuns: 0,
FailedRuns: 0,
StoppedRuns: 0,
Targets: []v1alpha1.TargetDetails{},
},
},
}
Expand Down Expand Up @@ -153,6 +154,7 @@ func PatchChaosResult(result *v1alpha1.ChaosResult, clients clients.ClientSets,
result.Status.ExperimentStatus.FailStep = resultDetails.FailStep
// for existing chaos result resource it will patch the label
result.ObjectMeta.Labels = chaosResultLabel
result.Status.History.Targets = chaosDetails.Targets
isAllProbePassed, result.Status.ProbeStatus = GetProbeStatus(resultDetails)
result.Status.ExperimentStatus.Verdict = resultDetails.Verdict

Expand Down Expand Up @@ -241,6 +243,7 @@ func updateHistory(result *v1alpha1.ChaosResult) {
PassedRuns: 0,
FailedRuns: 0,
StoppedRuns: 0,
Targets: []v1alpha1.TargetDetails{},
}
result.Status.History = history
}
Expand Down
8 changes: 6 additions & 2 deletions pkg/status/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,11 +56,15 @@ func AnnotatedApplicationsStatusCheck(appNs, appLabel, containerName string, tim
return errors.Errorf("Unable to find the pods with matching labels, err: %v", err)
}
for _, pod := range podList.Items {
isPodAnnotated, err := annotation.IsPodParentAnnotated(clients, pod, chaosDetails)
parentName, err := annotation.GetParentName(clients, pod, chaosDetails)
if err != nil {
return err
}
if isPodAnnotated {
isParentAnnotated, err := annotation.IsParentAnnotated(clients, parentName, chaosDetails)
if err != nil {
return err
}
if isParentAnnotated {
switch containerName {
case "":
for _, container := range pod.Status.ContainerStatuses {
Expand Down
2 changes: 2 additions & 0 deletions pkg/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ type ChaosDetails struct {
JobCleanupPolicy string
ProbeImagePullPolicy string
Randomness bool
Targets []v1alpha1.TargetDetails
ParentsResources []string
}

// AppDetails contains all the application related envs
Expand Down
Loading