Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore(probe): Adding the root cause into probe description #628

Merged
merged 1 commit into from
Jan 9, 2023
Merged

chore(probe): Adding the root cause into probe description #628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
50 changes: 34 additions & 16 deletions pkg/probe/cmdprobe.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,20 @@ func prepareCmdProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets,

// triggerInlineCmdProbe trigger the cmd probe and storing the output into the out buffer
func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.ResultDetails) error {
var description string

// It parse the templated command and return normal string
// It parses the templated command and return normal string
// if command doesn't have template, it will return the same command
probe.CmdProbeInputs.Command, err = parseCommand(probe.CmdProbeInputs.Command, resultDetails)
if err != nil {
return err
}

// running the cmd probe command and storing the output into the out buffer
// it will retry for some retry count, in each iterations of try it contains following things
// it will retry for some retry count, in each iteration of try it contains following things
// it contains a timeout per iteration of retry. if the timeout expires without success then it will go to next try
// for a timeout, it will run the command, if it fails wait for the iterval and again execute the command until timeout expires
return retry.Times(uint(probe.RunProperties.Retry)).
// for a timeout, it will run the command, if it fails wait for the interval and again execute the command until timeout expires
if err := retry.Times(uint(probe.RunProperties.Retry)).
Timeout(int64(probe.RunProperties.ProbeTimeout)).
Wait(time.Duration(probe.RunProperties.Interval) * time.Second).
TryWithTimeout(func(attempt uint) error {
Expand All @@ -79,7 +80,8 @@ func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.
}

rc := getAndIncrementRunCount(resultDetails, probe.Name)
if err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(out.String()), rc); err != nil {
description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(out.String()), rc)
if err != nil {
log.Errorf("the %v cmd probe has been Failed, err: %v", probe.Name, err)
return err
}
Expand All @@ -88,11 +90,17 @@ func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.
probes.ProbeArtifacts.Register = strings.TrimSpace(out.String())
resultDetails.ProbeArtifacts[probe.Name] = probes
return nil
})
}); err != nil {
return err
}

setProbeDescription(resultDetails, probe, description)
return nil
}

// triggerSourceCmdProbe trigger the cmd probe inside the external pod
func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails litmusexec.PodDetails, clients clients.ClientSets, resultDetails *types.ResultDetails) error {
var description string

// It parse the templated command and return normal string
// if command doesn't have template, it will return the same command
Expand All @@ -105,7 +113,7 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li
// it will retry for some retry count, in each iterations of try it contains following things
// it contains a timeout per iteration of retry. if the timeout expires without success then it will go to next try
// for a timeout, it will run the command, if it fails wait for the iterval and again execute the command until timeout expires
return retry.Times(uint(probe.RunProperties.Retry)).
if err := retry.Times(uint(probe.RunProperties.Retry)).
Timeout(int64(probe.RunProperties.ProbeTimeout)).
Wait(time.Duration(probe.RunProperties.Interval) * time.Second).
TryWithTimeout(func(attempt uint) error {
Expand All @@ -117,7 +125,7 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li
}

rc := getAndIncrementRunCount(resultDetails, probe.Name)
if err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(output), rc); err != nil {
if description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(output), rc); err != nil {
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
return err
}
Expand All @@ -126,7 +134,12 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li
probes.ProbeArtifacts.Register = strings.TrimSpace(output)
resultDetails.ProbeArtifacts[probe.Name] = probes
return nil
})
}); err != nil {
return err
}

setProbeDescription(resultDetails, probe, description)
return nil
}

// createProbePod creates an external pod with source image for the cmd probe
Expand Down Expand Up @@ -316,13 +329,14 @@ func triggerInlineContinuousCmdProbe(probe v1alpha1.ProbeAttributes, clients cli
// it marked the error for the probes, if any
loop:
for {
err = triggerInlineCmdProbe(probe, chaosresult)
err := triggerInlineCmdProbe(probe, chaosresult)
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
if err != nil {
err = addProbePhase(err, string(chaosDetails.Phase))
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand Down Expand Up @@ -373,6 +387,7 @@ loop:
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand Down Expand Up @@ -423,6 +438,7 @@ loop:
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand Down Expand Up @@ -465,6 +481,7 @@ loop:
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand All @@ -486,7 +503,7 @@ loop:

// validateResult validate the probe result to specified comparison operation
// it supports int, float, string operands
func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput string, rc int) error {
func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput string, rc int) (string, error) {

compare := cmp.RunCount(rc).
FirstValue(cmdOutput).
Expand All @@ -497,20 +514,21 @@ func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput str
switch strings.ToLower(comparator.Type) {
case "int":
if err = compare.CompareInt(cerrors.ErrorTypeCmdProbe); err != nil {
return err
return "", err
}
case "float":
if err = compare.CompareFloat(cerrors.ErrorTypeCmdProbe); err != nil {
return err
return "", err
}
case "string":
if err = compare.CompareString(cerrors.ErrorTypeCmdProbe); err != nil {
return err
return "", err
}
default:
return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Target: fmt.Sprintf("{name: %v}", probeName), Reason: fmt.Sprintf("comparator type '%s' not supported in the cmd probe", comparator.Type)}
return "", cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Target: fmt.Sprintf("{name: %v}", probeName), Reason: fmt.Sprintf("comparator type '%s' not supported in the cmd probe", comparator.Type)}
}
return nil
description := fmt.Sprintf("Probe responded with a valid output. Actual and Expected values are '%s' and '%s' respectively", cmdOutput, comparator.Value)
return description, nil
}

// preChaosCmdProbe trigger the cmd probe for prechaos phase
Expand Down
18 changes: 9 additions & 9 deletions pkg/probe/comparator/float.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,38 +24,38 @@ func (model Model) CompareFloat(errorCode cerrors.ErrorType) error {
switch model.operator {
case ">=":
if !obj.isGreaterorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than or equal to the Expected value: %v", obj.a, obj.b)}
}
case "<=":
if !obj.isLesserorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than or equal to the Expected value: %v", obj.a, obj.b)}
}
case ">":
if !obj.isGreater() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than the Expected value: %v", obj.a, obj.b)}
}
case "<":
if !obj.isLesser() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than the Expected value: %v", obj.a, obj.b)}
}
case "==":
if !obj.isEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not equal to the Expected value: %v", obj.a, obj.b)}
}
case "!=":
if !obj.isNotEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not Notequal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with the Expected value: %v", obj.a, obj.b)}
}
case "OneOf", "oneOf":
if !obj.isOneOf() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't matched with any of the expected values: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't matched any of the Expected values: %v", obj.a, obj.c)}
}
case "between", "Between":
if len(obj.c) < 2 {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("expected value: %v should contains both lower and upper limits", obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Expected value: %v should contains both the lower and upper limits", obj.c)}
}
if !obj.isBetween() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't lie in between expected range: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't lie in between the Expected range: %v", obj.a, obj.c)}
}
default:
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("criteria '%s' not supported in the probe", model.operator)}
Expand Down
18 changes: 9 additions & 9 deletions pkg/probe/comparator/integer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,38 +24,38 @@ func (model Model) CompareInt(errorCode cerrors.ErrorType) error {
switch model.operator {
case ">=":
if !obj.isGreaterorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than or equal to the Expected value: %v", obj.a, obj.b)}
}
case "<=":
if !obj.isLesserorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than or equal to the expected value: %v", obj.a, obj.b)}
}
case ">":
if !obj.isGreater() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than the Expected value: %v", obj.a, obj.b)}
}
case "<":
if !obj.isLesser() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than the Expected value: %v", obj.a, obj.b)}
}
case "==":
if !obj.isEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output Actual value: %v is not equal to the Expected value: %v", obj.a, obj.b)}
}
case "!=":
if !obj.isNotEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not NotEqual to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with the Expected value: %v", obj.a, obj.b)}
}
case "OneOf", "oneOf":
if !obj.isOneOf() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't matched with any of the expected values: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't matched any of the Expected values: %v", obj.a, obj.c)}
}
case "between", "Between":
if len(obj.c) < 2 {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("expected value: %v should contains both lower and upper limits", obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Expected value: %v should contains both lower and upper limits", obj.c)}
}
if !obj.isBetween() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't lie in between expected range: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't lie in between the Expected range: %v", obj.a, obj.c)}
}
default:
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("criteria '%s' not supported in the probe", model.operator)}
Expand Down
16 changes: 8 additions & 8 deletions pkg/probe/comparator/string.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,35 +24,35 @@ func (model Model) CompareString(errorCode cerrors.ErrorType) error {
switch model.operator {
case "equal", "Equal":
if !obj.isEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not equal to the Expected value: %v", obj.a, obj.b)}
}
case "notEqual", "NotEqual":
if !obj.isNotEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not Notequal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with the Expected value: %v", obj.a, obj.b)}
}
case "contains", "Contains":
if !obj.isContains() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't contains expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't contain Expected value: %v", obj.a, obj.b)}
}
case "matches", "Matches":
re, err := regexp.Compile(obj.b)
if err != nil {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("the probe regex '%s' is not a valid expression", obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("The probe regex '%s' is not a valid expression", obj.b)}
}
if !obj.isMatched(re) {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not matched with expected regex: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not matched with the Expected regex: %v", obj.a, obj.b)}
}
case "notMatches", "NotMatches":
re, err := regexp.Compile(obj.b)
if err != nil {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("the probe regex '%s' is not a valid expression", obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("The probe regex '%s' is not a valid expression", obj.b)}
}
if obj.isMatched(re) {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not NotMatched with expected regex: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with Expected regex: %v", obj.a, obj.b)}
}
case "oneOf", "OneOf":
if !obj.isOneOf() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't matched with any of the expected values: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't matched any of the the Expected values: %v", obj.a, obj.c)}
}
default:
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("criteria '%s' not supported in the probe", model.operator)}
Expand Down