Distinguish fatal vs non-fatal retryable errors

closes #1327

verification/epic.go

@@ -27,7 +27,7 @@ var VerifierPath string
 var ErrMissingSource = errors.New("MissingSource")
 var ErrVerifierStatus = errors.New("VerifierStatus")
 var ErrVideoUnavailable = errors.New("VideoUnavailable")
-var ErrAudioMismatch = Retryable{errors.New("AudioMismatch")}
+var ErrAudioMismatch = Fatal{Retryable{errors.New("AudioMismatch")}}
 var ErrTampered = Retryable{errors.New("Tampered")}
 
 type epicResolution struct {

verification/verify.go

@@ -29,7 +29,11 @@ type Retryable struct {
 	error
 }
 
-var ErrPixelMismatch = Retryable{errors.New("PixelMismatch")}
+type Fatal struct {
+	Retryable
+}
+
+var ErrPixelMismatch = Fatal{Retryable{errors.New("PixelMismatch")}}
 var ErrPixelsAbsent = errors.New("PixelsAbsent")
 var errPMCheckFailed = errors.New("PM Check Failed")
 
@@ -99,7 +103,6 @@ func (a byResScore) Less(i, j int) bool { return a[i].res.Score < a[j].res.Score
 type SegmentVerifier struct {
 	policy    *Policy
 	results   []SegmentVerifierResults
-	count     int
 	verifySig sigVerifyFn
 }
 
@@ -144,18 +147,17 @@ func (sv *SegmentVerifier) Verify(params *Params) (*Params, error) {
 		// Verification passed successfully, so use this set of params
 		return params, nil
 	}
-	sv.count++
 
-	// Append retryable errors to results
+	// Append non-fatal retryable errors to results
 	// The caller should terminate processing for non-retryable errors
-	if IsRetryable(err) {
+	if !IsFatal(err) && IsRetryable(err) {
 		r := SegmentVerifierResults{params: params, res: res}
 		sv.results = append(sv.results, r)
 	}
 
 	// Check for max retries
 	// If max hit, return best params so far
-	if sv.count > sv.policy.Retries {
+	if len(sv.results) > sv.policy.Retries {
 		if len(sv.results) <= 0 {
 			return nil, err
 		}
@@ -166,9 +168,14 @@ func (sv *SegmentVerifier) Verify(params *Params) (*Params, error) {
 	return nil, err
 }
 
+func IsFatal(err error) bool {
+	_, fatal := err.(Fatal)
+	return fatal
+}
+
 func IsRetryable(err error) bool {
 	_, retryable := err.(Retryable)
-	return retryable
+	return retryable || IsFatal(err)
 }
 
 func (sv *SegmentVerifier) sigVerification(params *Params) error {

verification/verify_test.go

@@ -15,6 +15,39 @@ import (
 	"github.com/livepeer/lpms/ffmpeg"
 )
 
+func TestFatalRetryable(t *testing.T) {
+	errNonRetryable := errors.New("NonRetryable")
+	assert.False(t, IsRetryable(errNonRetryable))
+	assert.False(t, IsFatal(errNonRetryable))
+
+	errRetryable := Retryable{errors.New("Retryable")}
+	assert.True(t, IsRetryable(errRetryable))
+	assert.False(t, IsFatal(errRetryable))
+
+	errFatalRetryable := Fatal{Retryable{errors.New("FatalRetryable")}}
+	assert.True(t, IsRetryable(errFatalRetryable))
+	assert.True(t, IsFatal(errFatalRetryable))
+
+	errNonFatalRetryable := Retryable{errors.New("NonFatalRetryable")}
+	assert.True(t, IsRetryable(errNonFatalRetryable))
+	assert.False(t, IsFatal(errNonFatalRetryable))
+
+	// check that these Retryable errors are classified
+	// correctly as Fatal / Non-fatal:
+
+	// check ErrAudioMismatch
+	assert.True(t, IsRetryable(ErrAudioMismatch))
+	assert.True(t, IsFatal(ErrAudioMismatch))
+
+	// check ErrPixelMismatch
+	assert.True(t, IsRetryable(ErrPixelMismatch))
+	assert.True(t, IsFatal(ErrPixelMismatch))
+
+	// check ErrTampered
+	assert.True(t, IsRetryable(ErrTampered))
+	assert.False(t, IsFatal(ErrTampered))
+}
+
 type stubVerifier struct {
 	results *Results
 	err     error
@@ -217,6 +250,12 @@ func TestVerify(t *testing.T) {
 	assert.Equal(ErrPixelsAbsent, err)
 	assert.Equal("mno", string(res.ManifestID)) // Still return best result
 
+	// A high score should still fail if pixel checking fails
+	verifier.results = &Results{Score: 21.0, Pixels: []int64{-1, -2}}
+	res, err = sv.Verify(&Params{ManifestID: "vws", Results: data})
+	assert.Equal(ErrPixelMismatch, err)
+	assert.Equal("mno", string(res.ManifestID))
+
 	// Check *not* retryable; should never get a result
 	sv = NewSegmentVerifier(&Policy{Verifier: verifier, Retries: 1}) // reset
 	verifier.err = errors.New("Stub Verifier Non-Retryable Error")