Distinguish fatal vs non-fatal retryable errors

closes #1327

verification/epic.go

@@ -27,7 +27,7 @@ var VerifierPath string
 var ErrMissingSource = errors.New("MissingSource")
 var ErrVerifierStatus = errors.New("VerifierStatus")
 var ErrVideoUnavailable = errors.New("VideoUnavailable")
-var ErrAudioMismatch = Retryable{errors.New("AudioMismatch")}
+var ErrAudioMismatch = Fatal{Retryable{errors.New("AudioMismatch")}}
 var ErrTampered = Retryable{errors.New("Tampered")}
 
 type epicResolution struct {

verification/verify.go

@@ -18,7 +18,11 @@ type Retryable struct {
 	error
 }
 
-var ErrPixelMismatch = Retryable{errors.New("PixelMismatch")}
+type Fatal struct {
+	Retryable
+}
+
+var ErrPixelMismatch = Fatal{Retryable{errors.New("PixelMismatch")}}
 var ErrPixelsAbsent = errors.New("PixelsAbsent")
 
 type Params struct {
@@ -85,7 +89,6 @@ func (a byResScore) Less(i, j int) bool { return a[i].res.Score < a[j].res.Score
 type SegmentVerifier struct {
 	policy  *Policy
 	results []SegmentVerifierResults
-	count   int
 }
 
 func NewSegmentVerifier(p *Policy) *SegmentVerifier {
@@ -127,18 +130,17 @@ func (sv *SegmentVerifier) Verify(params *Params) (*Params, error) {
 		// Verification passed successfully, so use this set of params
 		return params, nil
 	}
-	sv.count++
 
-	// Append retryable errors to results
+	// Append non-fatal retryable errors to results
 	// The caller should terminate processing for non-retryable errors
-	if IsRetryable(err) {
+	if !IsFatal(err) && IsRetryable(err) {
 		r := SegmentVerifierResults{params: params, res: res}
 		sv.results = append(sv.results, r)
 	}
 
 	// Check for max retries
 	// If max hit, return best params so far
-	if sv.count > sv.policy.Retries {
+	if len(sv.results) > sv.policy.Retries {
 		if len(sv.results) <= 0 {
 			return nil, err
 		}
@@ -149,7 +151,12 @@ func (sv *SegmentVerifier) Verify(params *Params) (*Params, error) {
 	return nil, err
 }
 
+func IsFatal(err error) bool {
+	_, fatal := err.(Fatal)
+	return fatal
+}
+
 func IsRetryable(err error) bool {
 	_, retryable := err.(Retryable)
-	return retryable
+	return retryable || IsFatal(err)
 }

verification/verify_test.go

@@ -9,6 +9,39 @@ import (
 	"github.com/livepeer/go-livepeer/net"
 )
 
+func TestFatalRetryable(t *testing.T) {
+	errNonRetryable := errors.New("NonRetryable")
+	assert.False(t, IsRetryable(errNonRetryable))
+	assert.False(t, IsFatal(errNonRetryable))
+
+	errRetryable := Retryable{errors.New("Retryable")}
+	assert.True(t, IsRetryable(errRetryable))
+	assert.False(t, IsFatal(errRetryable))
+
+	errFatalRetryable := Fatal{Retryable{errors.New("FatalRetryable")}}
+	assert.True(t, IsRetryable(errFatalRetryable))
+	assert.True(t, IsFatal(errFatalRetryable))
+
+	errNonFatalRetryable := Retryable{errors.New("NonFatalRetryable")}
+	assert.True(t, IsRetryable(errNonFatalRetryable))
+	assert.False(t, IsFatal(errNonFatalRetryable))
+
+	// check that these Retryable errors are classified
+	// correctly as Fatal / Non-fatal:
+
+	// check ErrAudioMismatch
+	assert.True(t, IsRetryable(ErrAudioMismatch))
+	assert.True(t, IsFatal(ErrAudioMismatch))
+
+	// check ErrPixelMismatch
+	assert.True(t, IsRetryable(ErrPixelMismatch))
+	assert.True(t, IsFatal(ErrPixelMismatch))
+
+	// check ErrTampered
+	assert.True(t, IsRetryable(ErrTampered))
+	assert.False(t, IsFatal(ErrTampered))
+}
+
 type stubVerifier struct {
 	results *Results
 	err     error