Skip to content

Commit

Permalink
chore(probe): Adding the root cause into probe description (#628)
Browse files Browse the repository at this point in the history 
Signed-off-by: Shubham Chaudhary <shubham.chaudhary@harness.io>
  • Loading branch information
@ispeakc0de
ispeakc0de committed Jan 9, 2023
1 parent dc92836 commit 3622f50
Show file tree
Hide file tree
Showing 9 changed files with 139 additions and 75 deletions.
50 changes: 34 additions & 16 deletions pkg/probe/cmdprobe.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,19 +53,20 @@ func prepareCmdProbe(probe v1alpha1.ProbeAttributes, clients clients.ClientSets,

// triggerInlineCmdProbe trigger the cmd probe and storing the output into the out buffer
func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.ResultDetails) error {
var description string

// It parse the templated command and return normal string
// It parses the templated command and return normal string
// if command doesn't have template, it will return the same command
probe.CmdProbeInputs.Command, err = parseCommand(probe.CmdProbeInputs.Command, resultDetails)
if err != nil {
return err
}

// running the cmd probe command and storing the output into the out buffer
// it will retry for some retry count, in each iterations of try it contains following things
// it will retry for some retry count, in each iteration of try it contains following things
// it contains a timeout per iteration of retry. if the timeout expires without success then it will go to next try
// for a timeout, it will run the command, if it fails wait for the iterval and again execute the command until timeout expires
return retry.Times(uint(probe.RunProperties.Retry)).
// for a timeout, it will run the command, if it fails wait for the interval and again execute the command until timeout expires
if err := retry.Times(uint(probe.RunProperties.Retry)).
Timeout(int64(probe.RunProperties.ProbeTimeout)).
Wait(time.Duration(probe.RunProperties.Interval) * time.Second).
TryWithTimeout(func(attempt uint) error {
Expand All @@ -79,7 +80,8 @@ func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.
}

rc := getAndIncrementRunCount(resultDetails, probe.Name)
if err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(out.String()), rc); err != nil {
description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(out.String()), rc)
if err != nil {
log.Errorf("the %v cmd probe has been Failed, err: %v", probe.Name, err)
return err
}
Expand All @@ -88,11 +90,17 @@ func triggerInlineCmdProbe(probe v1alpha1.ProbeAttributes, resultDetails *types.
probes.ProbeArtifacts.Register = strings.TrimSpace(out.String())
resultDetails.ProbeArtifacts[probe.Name] = probes
return nil
})
}); err != nil {
return err
}

setProbeDescription(resultDetails, probe, description)
return nil
}

// triggerSourceCmdProbe trigger the cmd probe inside the external pod
func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails litmusexec.PodDetails, clients clients.ClientSets, resultDetails *types.ResultDetails) error {
var description string

// It parse the templated command and return normal string
// if command doesn't have template, it will return the same command
Expand All @@ -105,7 +113,7 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li
// it will retry for some retry count, in each iterations of try it contains following things
// it contains a timeout per iteration of retry. if the timeout expires without success then it will go to next try
// for a timeout, it will run the command, if it fails wait for the iterval and again execute the command until timeout expires
return retry.Times(uint(probe.RunProperties.Retry)).
if err := retry.Times(uint(probe.RunProperties.Retry)).
Timeout(int64(probe.RunProperties.ProbeTimeout)).
Wait(time.Duration(probe.RunProperties.Interval) * time.Second).
TryWithTimeout(func(attempt uint) error {
Expand All @@ -117,7 +125,7 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li
}

rc := getAndIncrementRunCount(resultDetails, probe.Name)
if err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(output), rc); err != nil {
if description, err = validateResult(probe.CmdProbeInputs.Comparator, probe.Name, strings.TrimSpace(output), rc); err != nil {
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
return err
}
Expand All @@ -126,7 +134,12 @@ func triggerSourceCmdProbe(probe v1alpha1.ProbeAttributes, execCommandDetails li
probes.ProbeArtifacts.Register = strings.TrimSpace(output)
resultDetails.ProbeArtifacts[probe.Name] = probes
return nil
})
}); err != nil {
return err
}

setProbeDescription(resultDetails, probe, description)
return nil
}

// createProbePod creates an external pod with source image for the cmd probe
Expand Down Expand Up @@ -316,13 +329,14 @@ func triggerInlineContinuousCmdProbe(probe v1alpha1.ProbeAttributes, clients cli
// it marked the error for the probes, if any
loop:
for {
err = triggerInlineCmdProbe(probe, chaosresult)
err := triggerInlineCmdProbe(probe, chaosresult)
// record the error inside the probeDetails, we are maintaining a dedicated variable for the err, inside probeDetails
if err != nil {
err = addProbePhase(err, string(chaosDetails.Phase))
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand Down Expand Up @@ -373,6 +387,7 @@ loop:
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand Down Expand Up @@ -423,6 +438,7 @@ loop:
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand Down Expand Up @@ -465,6 +481,7 @@ loop:
for index := range chaosresult.ProbeDetails {
if chaosresult.ProbeDetails[index].Name == probe.Name {
chaosresult.ProbeDetails[index].IsProbeFailedWithError = err
chaosresult.ProbeDetails[index].Status.Description = getDescription(err)
log.Errorf("The %v cmd probe has been Failed, err: %v", probe.Name, err)
isExperimentFailed = true
break loop
Expand All @@ -486,7 +503,7 @@ loop:

// validateResult validate the probe result to specified comparison operation
// it supports int, float, string operands
func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput string, rc int) error {
func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput string, rc int) (string, error) {

compare := cmp.RunCount(rc).
FirstValue(cmdOutput).
Expand All @@ -497,20 +514,21 @@ func validateResult(comparator v1alpha1.ComparatorInfo, probeName, cmdOutput str
switch strings.ToLower(comparator.Type) {
case "int":
if err = compare.CompareInt(cerrors.ErrorTypeCmdProbe); err != nil {
return err
return "", err
}
case "float":
if err = compare.CompareFloat(cerrors.ErrorTypeCmdProbe); err != nil {
return err
return "", err
}
case "string":
if err = compare.CompareString(cerrors.ErrorTypeCmdProbe); err != nil {
return err
return "", err
}
default:
return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Target: fmt.Sprintf("{name: %v}", probeName), Reason: fmt.Sprintf("comparator type '%s' not supported in the cmd probe", comparator.Type)}
return "", cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Target: fmt.Sprintf("{name: %v}", probeName), Reason: fmt.Sprintf("comparator type '%s' not supported in the cmd probe", comparator.Type)}
}
return nil
description := fmt.Sprintf("Probe responded with a valid output. Actual and Expected values are '%s' and '%s' respectively", cmdOutput, comparator.Value)
return description, nil
}

// preChaosCmdProbe trigger the cmd probe for prechaos phase
Expand Down
18 changes: 9 additions & 9 deletions pkg/probe/comparator/float.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,38 +24,38 @@ func (model Model) CompareFloat(errorCode cerrors.ErrorType) error {
switch model.operator {
case ">=":
if !obj.isGreaterorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than or equal to the Expected value: %v", obj.a, obj.b)}
}
case "<=":
if !obj.isLesserorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than or equal to the Expected value: %v", obj.a, obj.b)}
}
case ">":
if !obj.isGreater() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than the Expected value: %v", obj.a, obj.b)}
}
case "<":
if !obj.isLesser() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than the Expected value: %v", obj.a, obj.b)}
}
case "==":
if !obj.isEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not equal to the Expected value: %v", obj.a, obj.b)}
}
case "!=":
if !obj.isNotEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not Notequal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with the Expected value: %v", obj.a, obj.b)}
}
case "OneOf", "oneOf":
if !obj.isOneOf() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't matched with any of the expected values: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't matched any of the Expected values: %v", obj.a, obj.c)}
}
case "between", "Between":
if len(obj.c) < 2 {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("expected value: %v should contains both lower and upper limits", obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Expected value: %v should contains both the lower and upper limits", obj.c)}
}
if !obj.isBetween() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't lie in between expected range: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't lie in between the Expected range: %v", obj.a, obj.c)}
}
default:
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("criteria '%s' not supported in the probe", model.operator)}
Expand Down
18 changes: 9 additions & 9 deletions pkg/probe/comparator/integer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,38 +24,38 @@ func (model Model) CompareInt(errorCode cerrors.ErrorType) error {
switch model.operator {
case ">=":
if !obj.isGreaterorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than or equal to the Expected value: %v", obj.a, obj.b)}
}
case "<=":
if !obj.isLesserorEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than or equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than or equal to the expected value: %v", obj.a, obj.b)}
}
case ">":
if !obj.isGreater() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not greater than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not greater than the Expected value: %v", obj.a, obj.b)}
}
case "<":
if !obj.isLesser() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not lesser than expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not lesser than the Expected value: %v", obj.a, obj.b)}
}
case "==":
if !obj.isEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output Actual value: %v is not equal to the Expected value: %v", obj.a, obj.b)}
}
case "!=":
if !obj.isNotEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not NotEqual to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with the Expected value: %v", obj.a, obj.b)}
}
case "OneOf", "oneOf":
if !obj.isOneOf() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't matched with any of the expected values: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't matched any of the Expected values: %v", obj.a, obj.c)}
}
case "between", "Between":
if len(obj.c) < 2 {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("expected value: %v should contains both lower and upper limits", obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Expected value: %v should contains both lower and upper limits", obj.c)}
}
if !obj.isBetween() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't lie in between expected range: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't lie in between the Expected range: %v", obj.a, obj.c)}
}
default:
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("criteria '%s' not supported in the probe", model.operator)}
Expand Down
16 changes: 8 additions & 8 deletions pkg/probe/comparator/string.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,35 +24,35 @@ func (model Model) CompareString(errorCode cerrors.ErrorType) error {
switch model.operator {
case "equal", "Equal":
if !obj.isEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not equal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not equal to the Expected value: %v", obj.a, obj.b)}
}
case "notEqual", "NotEqual":
if !obj.isNotEqual() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not Notequal to expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with the Expected value: %v", obj.a, obj.b)}
}
case "contains", "Contains":
if !obj.isContains() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't contains expected value: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't contain Expected value: %v", obj.a, obj.b)}
}
case "matches", "Matches":
re, err := regexp.Compile(obj.b)
if err != nil {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("the probe regex '%s' is not a valid expression", obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("The probe regex '%s' is not a valid expression", obj.b)}
}
if !obj.isMatched(re) {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not matched with expected regex: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v is not matched with the Expected regex: %v", obj.a, obj.b)}
}
case "notMatches", "NotMatches":
re, err := regexp.Compile(obj.b)
if err != nil {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("the probe regex '%s' is not a valid expression", obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("The probe regex '%s' is not a valid expression", obj.b)}
}
if obj.isMatched(re) {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v is not NotMatched with expected regex: %v", obj.a, obj.b)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v should not matched with Expected regex: %v", obj.a, obj.b)}
}
case "oneOf", "OneOf":
if !obj.isOneOf() {
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("actual value: %v doesn't matched with any of the expected values: %v", obj.a, obj.c)}
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("Probe responded with an invalid output. Actual value: %v doesn't matched any of the the Expected values: %v", obj.a, obj.c)}
}
default:
return cerrors.Error{ErrorCode: errorCode, Target: model.probeName, Reason: fmt.Sprintf("criteria '%s' not supported in the probe", model.operator)}
Expand Down

0 comments on commit 3622f50

Please sign in to comment.