Improve retry logic

caboose.go

@@ -85,11 +85,12 @@ const DefaultLoggingInterval = 5 * time.Second
 const DefaultSaturnOrchestratorRequestTimeout = 30 * time.Second
 
 const DefaultSaturnBlockRequestTimeout = 19 * time.Second
+
 const DefaultSaturnCarRequestTimeout = 30 * time.Minute
 const DefaultSaturnMirrorRequestTimeout = 30 * time.Second
 
 // default retries before failure unless overridden by MaxRetrievalAttempts
-const defaultMaxRetries = 3
+const defaultMaxRetries = 2
 
 // default percentage of requests to mirror for tracking how nodes perform unless overridden by MirrorFraction
 const defaultMirrorFraction = 0.01

fetcher.go

@@ -236,9 +236,9 @@ func (p *pool) fetchResource(ctx context.Context, from string, resource string,
 	// which is the amount of time without any NEW data from the server, but
 	// that can be added later. We need both because a slow trickle of data
 	// could take a large amount of time.
-	requestTimeout := DefaultSaturnCarRequestTimeout
+	requestTimeout := time.Duration(DefaultSaturnCarRequestTimeout.Milliseconds()/int64(attempt+1)) * time.Millisecond
 	if isBlockRequest {
-		requestTimeout = DefaultSaturnBlockRequestTimeout
+		requestTimeout = time.Duration(DefaultSaturnBlockRequestTimeout.Milliseconds()/int64(attempt+1)) * time.Millisecond
 	}
 
 	reqCtx, cancel := context.WithTimeout(ctx, requestTimeout)